{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import collections\n",
    "import gym\n",
    "import random\n",
    "import tensorflow.contrib.layers as layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ENV = \"CartPole-v0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MEMORY_SIZE = 10000\n",
    "EPISODES = 1000\n",
    "MAX_STEP = 500\n",
    "BATCH_SIZE = 32\n",
    "UPDATE_PERIOD = 200  # update target network parameters\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##built class for the DQN\n",
    "class DeepQNetwork():\n",
    "    def __init__(self , scope_main ,  env , sess=None , gamma = 0.8, epsilon = 0.8 , dueling = True , out_graph = False , out_dqn = True):\n",
    "        self.gamma = gamma\n",
    "        self.epsilon = epsilon\n",
    "        self.loss_his = []\n",
    "        \n",
    "        self.scope_main = scope_main\n",
    "        self.dueling = dueling\n",
    "        self.out_dqn = out_dqn\n",
    "        \n",
    "        self.action_dim = env.action_space.n\n",
    "        self.state_dim = env.observation_space.shape[0]\n",
    "        self.network()\n",
    "        self.sess = sess\n",
    "        self.sess.run(tf.global_variables_initializer())\n",
    "        tf.summary.FileWriter(\"DQN/summaries\" , sess.graph )\n",
    "        \n",
    "    # net_frame using for creating Q & target network\n",
    "    def net_frame(self , hiddens, inpt, num_actions, scope, hiddens_a , hiddens_v , reuse=None):\n",
    "        with tf.variable_scope(scope, reuse=reuse):\n",
    "            out = inpt  \n",
    "            for hidden in hiddens:\n",
    "                out = layers.fully_connected(out,  num_outputs=hidden, activation_fn=tf.nn.relu)\n",
    "            \n",
    "            if self.dueling == True :\n",
    "                # value_stream\n",
    "                with tf.variable_scope(\"value_stream\"):\n",
    "                    value = out\n",
    "                    for hidden in hiddens_v:\n",
    "                        value = layers.fully_connected(value, num_outputs= hidden , activation_fn=None) \n",
    "                    value = layers.fully_connected(value, num_outputs= 1 , activation_fn=None) \n",
    "\n",
    "                # advantage_stream\n",
    "                with tf.variable_scope(\"advantage_stream\"):\n",
    "                    advantage = out\n",
    "                    for hidden in hiddens_a:\n",
    "                        advantage = layers.fully_connected(advantage , num_outputs = hidden , activation_fn=None) \n",
    "                    advantage = layers.fully_connected(advantage , num_outputs= num_actions , activation_fn=None) \n",
    "\n",
    "                # aggregating_moudle\n",
    "                with tf.variable_scope(\"aggregating_moudle\"):\n",
    "                    q_out = value + advantage - tf.reduce_mean(advantage , axis = 1 , keep_dims = True )  # ***keep_dims\n",
    "\n",
    "            elif self.out_dqn:\n",
    "                with tf.variable_scope(\"dqn_out\"):\n",
    "                    q_out = layers.fully_connected(out, num_outputs=num_actions, activation_fn=None) \n",
    "\n",
    "            return q_out\n",
    "        \n",
    "    # create q_network & target_network     \n",
    "    def network(self):       \n",
    "        # q_network\n",
    "        self.inputs_q = tf.placeholder(dtype = tf.float32 , shape = [None , self.state_dim] , name = \"inputs_q\")\n",
    "        scope_var = \"q_network\"    \n",
    "        self.q_value = self.net_frame([64] , self.inputs_q , self.action_dim , scope_var , [20] , [20] , reuse = True )\n",
    "            \n",
    "        # target_network\n",
    "        self.inputs_target = tf.placeholder(dtype = tf.float32 , shape = [None , self.state_dim] , name = \"inputs_target\")\n",
    "        scope_tar = \"target_network\"    \n",
    "        self.q_target = self.net_frame([64] , self.inputs_target , self.action_dim , scope_tar , [20] , [20] )\n",
    "               \n",
    "        with tf.variable_scope(\"loss\"):\n",
    "            self.action = tf.placeholder(dtype = tf.int32 , shape = [ None ] , name = \"action\")\n",
    "            action_one_hot = tf.one_hot(self.action , self.action_dim )\n",
    "            q_action = tf.reduce_sum( tf.multiply(self.q_value , action_one_hot) , axis = 1 ) \n",
    "            \n",
    "            self.target =  tf.placeholder(dtype = tf.float32 , shape =  [None ] , name = \"target\")\n",
    "            self.loss = tf.reduce_mean( tf.square(q_action - self.target))\n",
    "\n",
    "        with tf.variable_scope(\"train\"):\n",
    "            optimizer = tf.train.RMSPropOptimizer(0.001)\n",
    "            self.train_op = optimizer.minimize(self.loss)    \n",
    "    \n",
    "    # training\n",
    "    def train(self , state , reward , action , state_next , done):\n",
    "        q , q_target = self.sess.run([self.q_value , self.q_target] , \n",
    "                                     feed_dict={self.inputs_q : state , self.inputs_target : state_next } )\n",
    "\n",
    "        q_target_best = np.max(q_target , axis = 1)\n",
    "        q_target_best_mask = ( 1.0 - done) * q_target_best\n",
    "        \n",
    "        target = reward + self.gamma * q_target_best_mask\n",
    "        \n",
    "        loss , _ = self.sess.run([self.loss , self.train_op] , \n",
    "                                 feed_dict={self.inputs_q: state , self.target:target , self.action:action} )   \n",
    "        self.loss_his.append(loss)\n",
    "        \n",
    "    # chose action\n",
    "    def chose_action(self , current_state):\n",
    "        current_state = current_state[np.newaxis , :]  #*** array dim: (xx,)  --> (1 , xx) ***\n",
    "        q = self.sess.run(self.q_value , feed_dict={self.inputs_q : current_state} )\n",
    "        \n",
    "        # e-greedy\n",
    "        if np.random.random() < self.epsilon:\n",
    "            action_chosen = np.random.randint(0 , self.action_dim)\n",
    "        else:\n",
    "            action_chosen = np.argmax(q)\n",
    "        \n",
    "        return action_chosen\n",
    "         \n",
    "    #upadate parmerters\n",
    "    def update_prmt(self):\n",
    "        q_prmts = tf.get_collection( tf.GraphKeys.GLOBAL_VARIABLES ,  self.scope_main + \"/q_network\"  )\n",
    "        target_prmts = tf.get_collection( tf.GraphKeys.GLOBAL_VARIABLES, self.scope_main + \"/target_network\"  )\n",
    "        self.sess.run( [tf.assign(t , q)for t,q in zip(target_prmts , q_prmts)])  #***\n",
    "        print(\"updating target-network parmeters...\")\n",
    "        \n",
    "    def decay_epsilon(self):\n",
    "        if self.epsilon > 0.03:\n",
    "            self.epsilon = self.epsilon - 0.02\n",
    "        \n",
    "    def greedy_action(self , current_state):\n",
    "        current_state = current_state[np.newaxis , :]  \n",
    "        q = self.sess.run(self.q_value , feed_dict={self.inputs_q : current_state} ) \n",
    "        action_greedy = np.argmax(q)\n",
    "        return action_greedy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# memory for momery replay\n",
    "memory = []\n",
    "Transition = collections.namedtuple(\"Transition\" , [\"state\", \"action\" , \"reward\" , \"next_state\" , \"done\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train( DQN , env ):\n",
    "    reward_his = []\n",
    "    all_reward = 0\n",
    "    step_his = []\n",
    "    update_iter = 0\n",
    "    for episode in range(EPISODES):\n",
    "        state = env.reset()\n",
    "        #training\n",
    "        for step in range(MAX_STEP):\n",
    "            action = DQN.chose_action(state)\n",
    "            next_state , reward , done , _ = env.step(action)\n",
    "            all_reward += reward \n",
    "\n",
    "            if len(memory) > MEMORY_SIZE:\n",
    "                memory.pop(0)\n",
    "            memory.append(Transition(state, action , reward , next_state , float(done)))\n",
    "\n",
    "            if len(memory) > BATCH_SIZE * 4:\n",
    "                batch_transition = random.sample(memory , BATCH_SIZE)\n",
    "                #***\n",
    "                batch_state, batch_action, batch_reward, batch_next_state, batch_done = map(np.array , zip(*batch_transition))  \n",
    "                DQN.train(state = batch_state ,\n",
    "                          reward = batch_reward , \n",
    "                          action = batch_action , \n",
    "                          state_next = batch_next_state,\n",
    "                          done = batch_done\n",
    "                         )\n",
    "                update_iter += 1\n",
    "\n",
    "            if update_iter % UPDATE_PERIOD == 0:\n",
    "                DQN.update_prmt()\n",
    "\n",
    "            if update_iter % 200 == 0:\n",
    "                DQN.decay_epsilon()\n",
    "\n",
    "            if done:\n",
    "                step_his.append(step)\n",
    "                reward_his.append(all_reward)\n",
    "                print(\"[episode= {} ] step = {}\".format(episode , step))\n",
    "                break\n",
    "\n",
    "            state = next_state\n",
    "            \n",
    "    loss_his = DQN.loss_his\n",
    "    return [step_his , reward_his , loss_his]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-06-25 19:05:12,701] Making new env: CartPole-v0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "[episode= 0 ] step = 12\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "[episode= 1 ] step = 15\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "[episode= 2 ] step = 14\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "[episode= 3 ] step = 8\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "[episode= 4 ] step = 8\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "[episode= 5 ] step = 9\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "[episode= 6 ] step = 9\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "[episode= 7 ] step = 9\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "[episode= 8 ] step = 8\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "[episode= 9 ] step = 8\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "[episode= 10 ] step = 9\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "updating target-network parmeters...\n",
      "[episode= 11 ] step = 8\n",
      "[episode= 12 ] step = 8\n",
      "[episode= 13 ] step = 8\n",
      "[episode= 14 ] step = 9\n",
      "[episode= 15 ] step = 7\n",
      "[episode= 16 ] step = 8\n",
      "[episode= 17 ] step = 7\n",
      "[episode= 18 ] step = 9\n",
      "[episode= 19 ] step = 8\n",
      "[episode= 20 ] step = 9\n",
      "[episode= 21 ] step = 8\n",
      "[episode= 22 ] step = 10\n",
      "[episode= 23 ] step = 8\n",
      "[episode= 24 ] step = 9\n",
      "[episode= 25 ] step = 8\n",
      "[episode= 26 ] step = 9\n",
      "[episode= 27 ] step = 8\n",
      "[episode= 28 ] step = 8\n",
      "[episode= 29 ] step = 8\n",
      "[episode= 30 ] step = 9\n",
      "[episode= 31 ] step = 11\n",
      "updating target-network parmeters...\n",
      "[episode= 32 ] step = 10\n",
      "[episode= 33 ] step = 8\n",
      "[episode= 34 ] step = 8\n",
      "[episode= 35 ] step = 7\n",
      "[episode= 36 ] step = 9\n",
      "[episode= 37 ] step = 7\n",
      "[episode= 38 ] step = 8\n",
      "[episode= 39 ] step = 10\n",
      "[episode= 40 ] step = 9\n",
      "[episode= 41 ] step = 8\n",
      "[episode= 42 ] step = 8\n",
      "[episode= 43 ] step = 8\n",
      "[episode= 44 ] step = 9\n",
      "[episode= 45 ] step = 10\n",
      "[episode= 46 ] step = 9\n",
      "[episode= 47 ] step = 8\n",
      "[episode= 48 ] step = 8\n",
      "[episode= 49 ] step = 9\n",
      "[episode= 50 ] step = 7\n",
      "[episode= 51 ] step = 9\n",
      "[episode= 52 ] step = 9\n",
      "[episode= 53 ] step = 9\n",
      "updating target-network parmeters...\n",
      "[episode= 54 ] step = 8\n",
      "[episode= 55 ] step = 9\n",
      "[episode= 56 ] step = 8\n",
      "[episode= 57 ] step = 9\n",
      "[episode= 58 ] step = 9\n",
      "[episode= 59 ] step = 9\n",
      "[episode= 60 ] step = 8\n",
      "[episode= 61 ] step = 9\n",
      "[episode= 62 ] step = 9\n",
      "[episode= 63 ] step = 8\n",
      "[episode= 64 ] step = 7\n",
      "[episode= 65 ] step = 8\n",
      "[episode= 66 ] step = 8\n",
      "[episode= 67 ] step = 7\n",
      "[episode= 68 ] step = 10\n",
      "[episode= 69 ] step = 9\n",
      "[episode= 70 ] step = 9\n",
      "[episode= 71 ] step = 8\n",
      "[episode= 72 ] step = 8\n",
      "[episode= 73 ] step = 8\n",
      "[episode= 74 ] step = 11\n",
      "updating target-network parmeters...\n",
      "[episode= 75 ] step = 8\n",
      "[episode= 76 ] step = 9\n",
      "[episode= 77 ] step = 10\n",
      "[episode= 78 ] step = 8\n",
      "[episode= 79 ] step = 9\n",
      "[episode= 80 ] step = 9\n",
      "[episode= 81 ] step = 8\n",
      "[episode= 82 ] step = 9\n",
      "[episode= 83 ] step = 10\n",
      "[episode= 84 ] step = 7\n",
      "[episode= 85 ] step = 8\n",
      "[episode= 86 ] step = 9\n",
      "[episode= 87 ] step = 11\n",
      "[episode= 88 ] step = 8\n",
      "[episode= 89 ] step = 9\n",
      "[episode= 90 ] step = 9\n",
      "[episode= 91 ] step = 7\n",
      "[episode= 92 ] step = 8\n",
      "[episode= 93 ] step = 9\n",
      "[episode= 94 ] step = 8\n",
      "updating target-network parmeters...\n",
      "[episode= 95 ] step = 8\n",
      "[episode= 96 ] step = 9\n",
      "[episode= 97 ] step = 7\n",
      "[episode= 98 ] step = 10\n",
      "[episode= 99 ] step = 8\n",
      "[episode= 100 ] step = 9\n",
      "[episode= 101 ] step = 9\n",
      "[episode= 102 ] step = 8\n",
      "[episode= 103 ] step = 9\n",
      "[episode= 104 ] step = 8\n",
      "[episode= 105 ] step = 9\n",
      "[episode= 106 ] step = 9\n",
      "[episode= 107 ] step = 9\n",
      "[episode= 108 ] step = 9\n",
      "[episode= 109 ] step = 9\n",
      "[episode= 110 ] step = 9\n",
      "[episode= 111 ] step = 9\n",
      "[episode= 112 ] step = 9\n",
      "[episode= 113 ] step = 9\n",
      "[episode= 114 ] step = 10\n",
      "[episode= 115 ] step = 9\n",
      "updating target-network parmeters...\n",
      "[episode= 116 ] step = 8\n",
      "[episode= 117 ] step = 9\n",
      "[episode= 118 ] step = 8\n",
      "[episode= 119 ] step = 9\n",
      "[episode= 120 ] step = 8\n",
      "[episode= 121 ] step = 8\n",
      "[episode= 122 ] step = 7\n",
      "[episode= 123 ] step = 10\n",
      "[episode= 124 ] step = 7\n",
      "[episode= 125 ] step = 9\n",
      "[episode= 126 ] step = 9\n",
      "[episode= 127 ] step = 7\n",
      "[episode= 128 ] step = 8\n",
      "[episode= 129 ] step = 9\n",
      "[episode= 130 ] step = 9\n",
      "[episode= 131 ] step = 7\n",
      "[episode= 132 ] step = 7\n",
      "[episode= 133 ] step = 9\n",
      "[episode= 134 ] step = 8\n",
      "[episode= 135 ] step = 9\n",
      "[episode= 136 ] step = 9\n",
      "updating target-network parmeters...\n",
      "[episode= 137 ] step = 8\n",
      "[episode= 138 ] step = 8\n",
      "[episode= 139 ] step = 7\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[episode= 140 ] step = 8\n",
      "[episode= 141 ] step = 8\n",
      "[episode= 142 ] step = 8\n",
      "[episode= 143 ] step = 9\n",
      "[episode= 144 ] step = 8\n",
      "[episode= 145 ] step = 9\n",
      "[episode= 146 ] step = 9\n",
      "[episode= 147 ] step = 8\n",
      "[episode= 148 ] step = 10\n",
      "[episode= 149 ] step = 11\n",
      "[episode= 150 ] step = 9\n",
      "[episode= 151 ] step = 9\n",
      "[episode= 152 ] step = 9\n",
      "[episode= 153 ] step = 8\n",
      "[episode= 154 ] step = 8\n",
      "[episode= 155 ] step = 16\n",
      "[episode= 156 ] step = 13\n",
      "updating target-network parmeters...\n",
      "[episode= 157 ] step = 11\n",
      "[episode= 158 ] step = 13\n",
      "[episode= 159 ] step = 13\n",
      "[episode= 160 ] step = 19\n",
      "[episode= 161 ] step = 14\n",
      "[episode= 162 ] step = 11\n",
      "[episode= 163 ] step = 12\n",
      "[episode= 164 ] step = 11\n",
      "[episode= 165 ] step = 16\n",
      "[episode= 166 ] step = 11\n",
      "[episode= 167 ] step = 15\n",
      "[episode= 168 ] step = 16\n",
      "[episode= 169 ] step = 13\n",
      "[episode= 170 ] step = 8\n",
      "updating target-network parmeters...\n",
      "[episode= 171 ] step = 9\n",
      "[episode= 172 ] step = 9\n",
      "[episode= 173 ] step = 7\n",
      "[episode= 174 ] step = 12\n",
      "[episode= 175 ] step = 9\n",
      "[episode= 176 ] step = 7\n",
      "[episode= 177 ] step = 8\n",
      "[episode= 178 ] step = 9\n",
      "[episode= 179 ] step = 20\n",
      "[episode= 180 ] step = 17\n",
      "[episode= 181 ] step = 17\n",
      "[episode= 182 ] step = 15\n",
      "[episode= 183 ] step = 10\n",
      "[episode= 184 ] step = 10\n",
      "[episode= 185 ] step = 14\n",
      "[episode= 186 ] step = 14\n",
      "updating target-network parmeters...\n",
      "[episode= 187 ] step = 14\n",
      "[episode= 188 ] step = 12\n",
      "[episode= 189 ] step = 11\n",
      "[episode= 190 ] step = 11\n",
      "[episode= 191 ] step = 12\n",
      "[episode= 192 ] step = 12\n",
      "[episode= 193 ] step = 12\n",
      "[episode= 194 ] step = 11\n",
      "[episode= 195 ] step = 16\n",
      "[episode= 196 ] step = 10\n",
      "[episode= 197 ] step = 13\n",
      "[episode= 198 ] step = 14\n",
      "[episode= 199 ] step = 11\n",
      "[episode= 200 ] step = 8\n",
      "[episode= 201 ] step = 7\n",
      "[episode= 202 ] step = 9\n",
      "updating target-network parmeters...\n",
      "[episode= 203 ] step = 23\n",
      "[episode= 204 ] step = 27\n",
      "[episode= 205 ] step = 20\n",
      "[episode= 206 ] step = 19\n",
      "[episode= 207 ] step = 31\n",
      "[episode= 208 ] step = 22\n",
      "[episode= 209 ] step = 21\n",
      "[episode= 210 ] step = 20\n",
      "updating target-network parmeters...\n",
      "[episode= 211 ] step = 19\n",
      "[episode= 212 ] step = 27\n",
      "[episode= 213 ] step = 21\n",
      "[episode= 214 ] step = 14\n",
      "[episode= 215 ] step = 13\n",
      "[episode= 216 ] step = 17\n",
      "[episode= 217 ] step = 20\n",
      "[episode= 218 ] step = 12\n",
      "[episode= 219 ] step = 23\n",
      "[episode= 220 ] step = 16\n",
      "[episode= 221 ] step = 20\n",
      "updating target-network parmeters...\n",
      "[episode= 222 ] step = 21\n",
      "[episode= 223 ] step = 20\n",
      "[episode= 224 ] step = 11\n",
      "[episode= 225 ] step = 17\n",
      "[episode= 226 ] step = 24\n",
      "[episode= 227 ] step = 29\n",
      "[episode= 228 ] step = 17\n",
      "[episode= 229 ] step = 17\n",
      "[episode= 230 ] step = 26\n",
      "updating target-network parmeters...\n",
      "[episode= 231 ] step = 33\n",
      "[episode= 232 ] step = 20\n",
      "[episode= 233 ] step = 25\n",
      "[episode= 234 ] step = 17\n",
      "[episode= 235 ] step = 34\n",
      "[episode= 236 ] step = 11\n",
      "[episode= 237 ] step = 21\n",
      "[episode= 238 ] step = 19\n",
      "updating target-network parmeters...\n",
      "[episode= 239 ] step = 21\n",
      "[episode= 240 ] step = 24\n",
      "[episode= 241 ] step = 14\n",
      "[episode= 242 ] step = 17\n",
      "[episode= 243 ] step = 20\n",
      "[episode= 244 ] step = 33\n",
      "[episode= 245 ] step = 13\n",
      "[episode= 246 ] step = 19\n",
      "[episode= 247 ] step = 29\n",
      "updating target-network parmeters...\n",
      "[episode= 248 ] step = 33\n",
      "[episode= 249 ] step = 24\n",
      "[episode= 250 ] step = 21\n",
      "[episode= 251 ] step = 37\n",
      "[episode= 252 ] step = 25\n",
      "[episode= 253 ] step = 19\n",
      "[episode= 254 ] step = 26\n",
      "[episode= 255 ] step = 19\n",
      "updating target-network parmeters...\n",
      "[episode= 256 ] step = 41\n",
      "[episode= 257 ] step = 24\n",
      "[episode= 258 ] step = 29\n",
      "[episode= 259 ] step = 52\n",
      "[episode= 260 ] step = 25\n",
      "[episode= 261 ] step = 26\n",
      "updating target-network parmeters...\n",
      "[episode= 262 ] step = 29\n",
      "[episode= 263 ] step = 59\n",
      "[episode= 264 ] step = 60\n",
      "[episode= 265 ] step = 54\n",
      "updating target-network parmeters...\n",
      "[episode= 266 ] step = 44\n",
      "[episode= 267 ] step = 54\n",
      "[episode= 268 ] step = 55\n",
      "[episode= 269 ] step = 37\n",
      "updating target-network parmeters...\n",
      "[episode= 270 ] step = 38\n",
      "[episode= 271 ] step = 38\n",
      "[episode= 272 ] step = 33\n",
      "[episode= 273 ] step = 33\n",
      "[episode= 274 ] step = 45\n",
      "updating target-network parmeters...\n",
      "[episode= 275 ] step = 43\n",
      "[episode= 276 ] step = 37\n",
      "[episode= 277 ] step = 43\n",
      "[episode= 278 ] step = 31\n",
      "[episode= 279 ] step = 46\n",
      "updating target-network parmeters...\n",
      "[episode= 280 ] step = 41\n",
      "[episode= 281 ] step = 35\n",
      "[episode= 282 ] step = 57\n",
      "[episode= 283 ] step = 35\n",
      "updating target-network parmeters...\n",
      "[episode= 284 ] step = 43\n",
      "[episode= 285 ] step = 58\n",
      "[episode= 286 ] step = 33\n",
      "[episode= 287 ] step = 55\n",
      "updating target-network parmeters...\n",
      "[episode= 288 ] step = 45\n",
      "[episode= 289 ] step = 40\n",
      "[episode= 290 ] step = 30\n",
      "[episode= 291 ] step = 34\n",
      "[episode= 292 ] step = 34\n",
      "[episode= 293 ] step = 38\n",
      "updating target-network parmeters...\n",
      "[episode= 294 ] step = 32\n",
      "[episode= 295 ] step = 26\n",
      "[episode= 296 ] step = 36\n",
      "[episode= 297 ] step = 26\n",
      "[episode= 298 ] step = 34\n",
      "[episode= 299 ] step = 26\n",
      "updating target-network parmeters...\n",
      "[episode= 300 ] step = 36\n",
      "[episode= 301 ] step = 30\n",
      "[episode= 302 ] step = 40\n",
      "[episode= 303 ] step = 26\n",
      "[episode= 304 ] step = 41\n",
      "[episode= 305 ] step = 35\n",
      "updating target-network parmeters...\n",
      "[episode= 306 ] step = 28\n",
      "[episode= 307 ] step = 30\n",
      "[episode= 308 ] step = 33\n",
      "[episode= 309 ] step = 30\n",
      "[episode= 310 ] step = 35\n",
      "[episode= 311 ] step = 30\n",
      "updating target-network parmeters...\n",
      "[episode= 312 ] step = 43\n",
      "[episode= 313 ] step = 32\n",
      "[episode= 314 ] step = 38\n",
      "[episode= 315 ] step = 32\n",
      "[episode= 316 ] step = 31\n",
      "[episode= 317 ] step = 25\n",
      "updating target-network parmeters...\n",
      "[episode= 318 ] step = 34\n",
      "[episode= 319 ] step = 30\n",
      "[episode= 320 ] step = 34\n",
      "[episode= 321 ] step = 40\n",
      "[episode= 322 ] step = 41\n",
      "updating target-network parmeters...\n",
      "[episode= 323 ] step = 44\n",
      "[episode= 324 ] step = 39\n",
      "[episode= 325 ] step = 29\n",
      "[episode= 326 ] step = 36\n",
      "[episode= 327 ] step = 35\n",
      "[episode= 328 ] step = 34\n",
      "updating target-network parmeters...\n",
      "[episode= 329 ] step = 26\n",
      "[episode= 330 ] step = 33\n",
      "[episode= 331 ] step = 36\n",
      "[episode= 332 ] step = 30\n",
      "[episode= 333 ] step = 31\n",
      "[episode= 334 ] step = 26\n",
      "updating target-network parmeters...\n",
      "[episode= 335 ] step = 29\n",
      "[episode= 336 ] step = 33\n",
      "[episode= 337 ] step = 34\n",
      "[episode= 338 ] step = 29\n",
      "[episode= 339 ] step = 29\n",
      "[episode= 340 ] step = 36\n",
      "updating target-network parmeters...\n",
      "[episode= 341 ] step = 42\n",
      "[episode= 342 ] step = 47\n",
      "[episode= 343 ] step = 42\n",
      "[episode= 344 ] step = 59\n",
      "updating target-network parmeters...\n",
      "[episode= 345 ] step = 44\n",
      "[episode= 346 ] step = 31\n",
      "[episode= 347 ] step = 51\n",
      "[episode= 348 ] step = 36\n",
      "[episode= 349 ] step = 53\n",
      "updating target-network parmeters...\n",
      "[episode= 350 ] step = 33\n",
      "[episode= 351 ] step = 46\n",
      "[episode= 352 ] step = 45\n",
      "[episode= 353 ] step = 35\n",
      "[episode= 354 ] step = 32\n",
      "updating target-network parmeters...\n",
      "[episode= 355 ] step = 50\n",
      "[episode= 356 ] step = 45\n",
      "[episode= 357 ] step = 59\n",
      "updating target-network parmeters...\n",
      "[episode= 358 ] step = 63\n",
      "[episode= 359 ] step = 46\n",
      "[episode= 360 ] step = 31\n",
      "[episode= 361 ] step = 68\n",
      "updating target-network parmeters...\n",
      "[episode= 362 ] step = 70\n",
      "[episode= 363 ] step = 60\n",
      "[episode= 364 ] step = 86\n",
      "updating target-network parmeters...\n",
      "[episode= 365 ] step = 44\n",
      "[episode= 366 ] step = 70\n",
      "[episode= 367 ] step = 68\n",
      "updating target-network parmeters...\n",
      "[episode= 368 ] step = 69\n",
      "[episode= 369 ] step = 66\n",
      "updating target-network parmeters...\n",
      "[episode= 370 ] step = 131\n",
      "[episode= 371 ] step = 113\n",
      "updating target-network parmeters...\n",
      "[episode= 372 ] step = 97\n",
      "[episode= 373 ] step = 89\n",
      "updating target-network parmeters...\n",
      "[episode= 374 ] step = 79\n",
      "[episode= 375 ] step = 104\n",
      "updating target-network parmeters...\n",
      "[episode= 376 ] step = 110\n",
      "[episode= 377 ] step = 141\n",
      "updating target-network parmeters...\n",
      "[episode= 378 ] step = 112\n",
      "updating target-network parmeters...\n",
      "[episode= 379 ] step = 136\n",
      "[episode= 380 ] step = 127\n",
      "updating target-network parmeters...\n",
      "[episode= 381 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode= 382 ] step = 158\n",
      "[episode= 383 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode= 384 ] step = 141\n",
      "updating target-network parmeters...\n",
      "[episode= 385 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode= 386 ] step = 182\n",
      "[episode= 387 ] step = 132\n",
      "updating target-network parmeters...\n",
      "[episode= 388 ] step = 131\n",
      "updating target-network parmeters...\n",
      "[episode= 389 ] step = 132\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[episode= 390 ] step = 150\n",
      "updating target-network parmeters...\n",
      "[episode= 391 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 392 ] step = 194\n",
      "updating target-network parmeters...\n",
      "[episode= 393 ] step = 164\n",
      "updating target-network parmeters...\n",
      "[episode= 394 ] step = 142\n",
      "updating target-network parmeters...\n",
      "[episode= 395 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 396 ] step = 148\n",
      "updating target-network parmeters...\n",
      "[episode= 397 ] step = 181\n",
      "updating target-network parmeters...\n",
      "[episode= 398 ] step = 180\n",
      "[episode= 399 ] step = 195\n",
      "updating target-network parmeters...\n",
      "[episode= 400 ] step = 156\n",
      "updating target-network parmeters...\n",
      "[episode= 401 ] step = 153\n",
      "updating target-network parmeters...\n",
      "[episode= 402 ] step = 143\n",
      "[episode= 403 ] step = 142\n",
      "updating target-network parmeters...\n",
      "[episode= 404 ] step = 150\n",
      "updating target-network parmeters...\n",
      "[episode= 405 ] step = 146\n",
      "updating target-network parmeters...\n",
      "[episode= 406 ] step = 173\n",
      "updating target-network parmeters...\n",
      "[episode= 407 ] step = 139\n",
      "[episode= 408 ] step = 143\n",
      "updating target-network parmeters...\n",
      "[episode= 409 ] step = 159\n",
      "updating target-network parmeters...\n",
      "[episode= 410 ] step = 162\n",
      "updating target-network parmeters...\n",
      "[episode= 411 ] step = 145\n",
      "[episode= 412 ] step = 147\n",
      "updating target-network parmeters...\n",
      "[episode= 413 ] step = 132\n",
      "updating target-network parmeters...\n",
      "[episode= 414 ] step = 134\n",
      "[episode= 415 ] step = 127\n",
      "updating target-network parmeters...\n",
      "[episode= 416 ] step = 154\n",
      "updating target-network parmeters...\n",
      "[episode= 417 ] step = 153\n",
      "updating target-network parmeters...\n",
      "[episode= 418 ] step = 149\n",
      "[episode= 419 ] step = 149\n",
      "updating target-network parmeters...\n",
      "[episode= 420 ] step = 155\n",
      "updating target-network parmeters...\n",
      "[episode= 421 ] step = 150\n",
      "updating target-network parmeters...\n",
      "[episode= 422 ] step = 144\n",
      "[episode= 423 ] step = 160\n",
      "updating target-network parmeters...\n",
      "[episode= 424 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 425 ] step = 190\n",
      "updating target-network parmeters...\n",
      "[episode= 426 ] step = 172\n",
      "updating target-network parmeters...\n",
      "[episode= 427 ] step = 156\n",
      "updating target-network parmeters...\n",
      "[episode= 428 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 429 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 430 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 431 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 432 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 433 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 434 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 435 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 436 ] step = 186\n",
      "updating target-network parmeters...\n",
      "[episode= 437 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 438 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 439 ] step = 192\n",
      "updating target-network parmeters...\n",
      "[episode= 440 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 441 ] step = 160\n",
      "updating target-network parmeters...\n",
      "[episode= 442 ] step = 174\n",
      "updating target-network parmeters...\n",
      "[episode= 443 ] step = 199\n",
      "[episode= 444 ] step = 167\n",
      "updating target-network parmeters...\n",
      "[episode= 445 ] step = 179\n",
      "updating target-network parmeters...\n",
      "[episode= 446 ] step = 161\n",
      "updating target-network parmeters...\n",
      "[episode= 447 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 448 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 449 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 450 ] step = 168\n",
      "updating target-network parmeters...\n",
      "[episode= 451 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 452 ] step = 99\n",
      "updating target-network parmeters...\n",
      "[episode= 453 ] step = 199\n",
      "[episode= 454 ] step = 182\n",
      "updating target-network parmeters...\n",
      "[episode= 455 ] step = 185\n",
      "updating target-network parmeters...\n",
      "[episode= 456 ] step = 168\n",
      "updating target-network parmeters...\n",
      "[episode= 457 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 458 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 459 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 460 ] step = 175\n",
      "updating target-network parmeters...\n",
      "[episode= 461 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 462 ] step = 161\n",
      "updating target-network parmeters...\n",
      "[episode= 463 ] step = 196\n",
      "updating target-network parmeters...\n",
      "[episode= 464 ] step = 133\n",
      "[episode= 465 ] step = 142\n",
      "updating target-network parmeters...\n",
      "[episode= 466 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 467 ] step = 155\n",
      "updating target-network parmeters...\n",
      "[episode= 468 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 469 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 470 ] step = 190\n",
      "updating target-network parmeters...\n",
      "[episode= 471 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 472 ] step = 177\n",
      "updating target-network parmeters...\n",
      "[episode= 473 ] step = 171\n",
      "updating target-network parmeters...\n",
      "[episode= 474 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 475 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 476 ] step = 199\n",
      "[episode= 477 ] step = 140\n",
      "updating target-network parmeters...\n",
      "[episode= 478 ] step = 115\n",
      "updating target-network parmeters...\n",
      "[episode= 479 ] step = 140\n",
      "[episode= 480 ] step = 9\n",
      "[episode= 481 ] step = 76\n",
      "updating target-network parmeters...\n",
      "[episode= 482 ] step = 102\n",
      "[episode= 483 ] step = 10\n",
      "[episode= 484 ] step = 10\n",
      "[episode= 485 ] step = 10\n",
      "[episode= 486 ] step = 13\n",
      "[episode= 487 ] step = 9\n",
      "[episode= 488 ] step = 12\n",
      "[episode= 489 ] step = 13\n",
      "[episode= 490 ] step = 11\n",
      "[episode= 491 ] step = 9\n",
      "[episode= 492 ] step = 14\n",
      "[episode= 493 ] step = 10\n",
      "[episode= 494 ] step = 10\n",
      "[episode= 495 ] step = 15\n",
      "updating target-network parmeters...\n",
      "[episode= 496 ] step = 13\n",
      "[episode= 497 ] step = 10\n",
      "[episode= 498 ] step = 7\n",
      "[episode= 499 ] step = 10\n",
      "[episode= 500 ] step = 14\n",
      "[episode= 501 ] step = 37\n",
      "[episode= 502 ] step = 82\n",
      "[episode= 503 ] step = 8\n",
      "[episode= 504 ] step = 10\n",
      "updating target-network parmeters...\n",
      "[episode= 505 ] step = 8\n",
      "[episode= 506 ] step = 10\n",
      "[episode= 507 ] step = 10\n",
      "[episode= 508 ] step = 9\n",
      "[episode= 509 ] step = 8\n",
      "[episode= 510 ] step = 9\n",
      "[episode= 511 ] step = 9\n",
      "[episode= 512 ] step = 9\n",
      "[episode= 513 ] step = 10\n",
      "[episode= 514 ] step = 8\n",
      "[episode= 515 ] step = 7\n",
      "[episode= 516 ] step = 10\n",
      "[episode= 517 ] step = 15\n",
      "[episode= 518 ] step = 9\n",
      "[episode= 519 ] step = 9\n",
      "[episode= 520 ] step = 10\n",
      "[episode= 521 ] step = 14\n",
      "[episode= 522 ] step = 9\n",
      "[episode= 523 ] step = 8\n",
      "updating target-network parmeters...\n",
      "[episode= 524 ] step = 8\n",
      "[episode= 525 ] step = 8\n",
      "[episode= 526 ] step = 12\n",
      "[episode= 527 ] step = 9\n",
      "[episode= 528 ] step = 17\n",
      "[episode= 529 ] step = 10\n",
      "[episode= 530 ] step = 10\n",
      "[episode= 531 ] step = 10\n",
      "[episode= 532 ] step = 10\n",
      "[episode= 533 ] step = 9\n",
      "[episode= 534 ] step = 16\n",
      "[episode= 535 ] step = 10\n",
      "[episode= 536 ] step = 13\n",
      "[episode= 537 ] step = 8\n",
      "[episode= 538 ] step = 8\n",
      "[episode= 539 ] step = 9\n",
      "[episode= 540 ] step = 9\n",
      "updating target-network parmeters...\n",
      "[episode= 541 ] step = 23\n",
      "[episode= 542 ] step = 75\n",
      "[episode= 543 ] step = 57\n",
      "[episode= 544 ] step = 7\n",
      "[episode= 545 ] step = 8\n",
      "[episode= 546 ] step = 10\n",
      "[episode= 547 ] step = 12\n",
      "[episode= 548 ] step = 11\n",
      "updating target-network parmeters...\n",
      "[episode= 549 ] step = 182\n",
      "updating target-network parmeters...\n",
      "[episode= 550 ] step = 136\n",
      "[episode= 551 ] step = 22\n",
      "updating target-network parmeters...\n",
      "[episode= 552 ] step = 105\n",
      "[episode= 553 ] step = 147\n",
      "updating target-network parmeters...\n",
      "[episode= 554 ] step = 139\n",
      "updating target-network parmeters...\n",
      "[episode= 555 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 556 ] step = 147\n",
      "updating target-network parmeters...\n",
      "[episode= 557 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 558 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 559 ] step = 199\n",
      "[episode= 560 ] step = 9\n",
      "updating target-network parmeters...\n",
      "[episode= 561 ] step = 190\n",
      "updating target-network parmeters...\n",
      "[episode= 562 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 563 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 564 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 565 ] step = 199\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "updating target-network parmeters...\n",
      "[episode= 566 ] step = 188\n",
      "updating target-network parmeters...\n",
      "[episode= 567 ] step = 187\n",
      "updating target-network parmeters...\n",
      "[episode= 568 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 569 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 570 ] step = 199\n",
      "[episode= 571 ] step = 131\n",
      "updating target-network parmeters...\n",
      "[episode= 572 ] step = 158\n",
      "updating target-network parmeters...\n",
      "[episode= 573 ] step = 122\n",
      "updating target-network parmeters...\n",
      "[episode= 574 ] step = 126\n",
      "[episode= 575 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 576 ] step = 120\n",
      "[episode= 577 ] step = 122\n",
      "updating target-network parmeters...\n",
      "[episode= 578 ] step = 131\n",
      "updating target-network parmeters...\n",
      "[episode= 579 ] step = 127\n",
      "[episode= 580 ] step = 142\n",
      "updating target-network parmeters...\n",
      "[episode= 581 ] step = 135\n",
      "updating target-network parmeters...\n",
      "[episode= 582 ] step = 182\n",
      "updating target-network parmeters...\n",
      "[episode= 583 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 584 ] step = 153\n",
      "updating target-network parmeters...\n",
      "[episode= 585 ] step = 160\n",
      "[episode= 586 ] step = 140\n",
      "updating target-network parmeters...\n",
      "[episode= 587 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 588 ] step = 188\n",
      "updating target-network parmeters...\n",
      "[episode= 589 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 590 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 591 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 592 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 593 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 594 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 595 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 596 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 597 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 598 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 599 ] step = 174\n",
      "updating target-network parmeters...\n",
      "[episode= 600 ] step = 121\n",
      "[episode= 601 ] step = 143\n",
      "updating target-network parmeters...\n",
      "[episode= 602 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 603 ] step = 137\n",
      "updating target-network parmeters...\n",
      "[episode= 604 ] step = 185\n",
      "updating target-network parmeters...\n",
      "[episode= 605 ] step = 140\n",
      "updating target-network parmeters...\n",
      "[episode= 606 ] step = 171\n",
      "updating target-network parmeters...\n",
      "[episode= 607 ] step = 192\n",
      "[episode= 608 ] step = 152\n",
      "updating target-network parmeters...\n",
      "[episode= 609 ] step = 161\n",
      "updating target-network parmeters...\n",
      "[episode= 610 ] step = 163\n",
      "updating target-network parmeters...\n",
      "[episode= 611 ] step = 131\n",
      "[episode= 612 ] step = 143\n",
      "updating target-network parmeters...\n",
      "[episode= 613 ] step = 129\n",
      "updating target-network parmeters...\n",
      "[episode= 614 ] step = 126\n",
      "[episode= 615 ] step = 135\n",
      "updating target-network parmeters...\n",
      "[episode= 616 ] step = 170\n",
      "updating target-network parmeters...\n",
      "[episode= 617 ] step = 164\n",
      "updating target-network parmeters...\n",
      "[episode= 618 ] step = 126\n",
      "[episode= 619 ] step = 129\n",
      "updating target-network parmeters...\n",
      "[episode= 620 ] step = 112\n",
      "[episode= 621 ] step = 124\n",
      "updating target-network parmeters...\n",
      "[episode= 622 ] step = 145\n",
      "updating target-network parmeters...\n",
      "[episode= 623 ] step = 113\n",
      "[episode= 624 ] step = 135\n",
      "[episode= 625 ] step = 16\n",
      "updating target-network parmeters...\n",
      "[episode= 626 ] step = 113\n",
      "[episode= 627 ] step = 11\n",
      "updating target-network parmeters...\n",
      "[episode= 628 ] step = 129\n",
      "[episode= 629 ] step = 130\n",
      "updating target-network parmeters...\n",
      "[episode= 630 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 631 ] step = 141\n",
      "[episode= 632 ] step = 129\n",
      "updating target-network parmeters...\n",
      "[episode= 633 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 634 ] step = 178\n",
      "updating target-network parmeters...\n",
      "[episode= 635 ] step = 178\n",
      "updating target-network parmeters...\n",
      "[episode= 636 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 637 ] step = 167\n",
      "updating target-network parmeters...\n",
      "[episode= 638 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 639 ] step = 199\n",
      "[episode= 640 ] step = 133\n",
      "updating target-network parmeters...\n",
      "[episode= 641 ] step = 150\n",
      "updating target-network parmeters...\n",
      "[episode= 642 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 643 ] step = 158\n",
      "updating target-network parmeters...\n",
      "[episode= 644 ] step = 190\n",
      "[episode= 645 ] step = 122\n",
      "updating target-network parmeters...\n",
      "[episode= 646 ] step = 96\n",
      "[episode= 647 ] step = 107\n",
      "updating target-network parmeters...\n",
      "[episode= 648 ] step = 184\n",
      "updating target-network parmeters...\n",
      "[episode= 649 ] step = 124\n",
      "updating target-network parmeters...\n",
      "[episode= 650 ] step = 119\n",
      "[episode= 651 ] step = 139\n",
      "updating target-network parmeters...\n",
      "[episode= 652 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 653 ] step = 132\n",
      "updating target-network parmeters...\n",
      "[episode= 654 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 655 ] step = 136\n",
      "[episode= 656 ] step = 128\n",
      "updating target-network parmeters...\n",
      "[episode= 657 ] step = 194\n",
      "updating target-network parmeters...\n",
      "[episode= 658 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 659 ] step = 160\n",
      "updating target-network parmeters...\n",
      "[episode= 660 ] step = 93\n",
      "[episode= 661 ] step = 150\n",
      "updating target-network parmeters...\n",
      "[episode= 662 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode= 663 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 664 ] step = 114\n",
      "[episode= 665 ] step = 130\n",
      "updating target-network parmeters...\n",
      "[episode= 666 ] step = 173\n",
      "[episode= 667 ] step = 61\n",
      "updating target-network parmeters...\n",
      "[episode= 668 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 669 ] step = 128\n",
      "updating target-network parmeters...\n",
      "[episode= 670 ] step = 118\n",
      "[episode= 671 ] step = 148\n",
      "updating target-network parmeters...\n",
      "[episode= 672 ] step = 140\n",
      "updating target-network parmeters...\n",
      "[episode= 673 ] step = 131\n",
      "[episode= 674 ] step = 61\n",
      "[episode= 675 ] step = 64\n",
      "updating target-network parmeters...\n",
      "[episode= 676 ] step = 33\n",
      "[episode= 677 ] step = 79\n",
      "[episode= 678 ] step = 74\n",
      "updating target-network parmeters...\n",
      "[episode= 679 ] step = 52\n",
      "[episode= 680 ] step = 60\n",
      "[episode= 681 ] step = 36\n",
      "[episode= 682 ] step = 43\n",
      "updating target-network parmeters...\n",
      "[episode= 683 ] step = 39\n",
      "[episode= 684 ] step = 50\n",
      "[episode= 685 ] step = 43\n",
      "[episode= 686 ] step = 36\n",
      "[episode= 687 ] step = 34\n",
      "[episode= 688 ] step = 27\n",
      "updating target-network parmeters...\n",
      "[episode= 689 ] step = 42\n",
      "[episode= 690 ] step = 63\n",
      "updating target-network parmeters...\n",
      "[episode= 691 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 692 ] step = 129\n",
      "updating target-network parmeters...\n",
      "[episode= 693 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 694 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 695 ] step = 199\n",
      "[episode= 696 ] step = 101\n",
      "updating target-network parmeters...\n",
      "[episode= 697 ] step = 109\n",
      "[episode= 698 ] step = 114\n",
      "updating target-network parmeters...\n",
      "[episode= 699 ] step = 106\n",
      "updating target-network parmeters...\n",
      "[episode= 700 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 701 ] step = 148\n",
      "[episode= 702 ] step = 155\n",
      "updating target-network parmeters...\n",
      "[episode= 703 ] step = 157\n",
      "[episode= 704 ] step = 14\n",
      "updating target-network parmeters...\n",
      "[episode= 705 ] step = 54\n",
      "[episode= 706 ] step = 93\n",
      "[episode= 707 ] step = 15\n",
      "[episode= 708 ] step = 32\n",
      "[episode= 709 ] step = 36\n",
      "updating target-network parmeters...\n",
      "[episode= 710 ] step = 35\n",
      "[episode= 711 ] step = 101\n",
      "[episode= 712 ] step = 79\n",
      "updating target-network parmeters...\n",
      "[episode= 713 ] step = 76\n",
      "updating target-network parmeters...\n",
      "[episode= 714 ] step = 154\n",
      "[episode= 715 ] step = 122\n",
      "updating target-network parmeters...\n",
      "[episode= 716 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 717 ] step = 82\n",
      "[episode= 718 ] step = 124\n",
      "updating target-network parmeters...\n",
      "[episode= 719 ] step = 86\n",
      "[episode= 720 ] step = 68\n",
      "[episode= 721 ] step = 62\n",
      "updating target-network parmeters...\n",
      "[episode= 722 ] step = 85\n",
      "[episode= 723 ] step = 82\n",
      "updating target-network parmeters...\n",
      "[episode= 724 ] step = 77\n",
      "[episode= 725 ] step = 81\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[episode= 726 ] step = 87\n",
      "updating target-network parmeters...\n",
      "[episode= 727 ] step = 137\n",
      "updating target-network parmeters...\n",
      "[episode= 728 ] step = 119\n",
      "updating target-network parmeters...\n",
      "[episode= 729 ] step = 151\n",
      "[episode= 730 ] step = 163\n",
      "updating target-network parmeters...\n",
      "[episode= 731 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 732 ] step = 184\n",
      "updating target-network parmeters...\n",
      "[episode= 733 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 734 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 735 ] step = 52\n",
      "[episode= 736 ] step = 87\n",
      "updating target-network parmeters...\n",
      "[episode= 737 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 738 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 739 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 740 ] step = 199\n",
      "[episode= 741 ] step = 97\n",
      "updating target-network parmeters...\n",
      "[episode= 742 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 743 ] step = 118\n",
      "updating target-network parmeters...\n",
      "[episode= 744 ] step = 131\n",
      "updating target-network parmeters...\n",
      "[episode= 745 ] step = 186\n",
      "[episode= 746 ] step = 157\n",
      "updating target-network parmeters...\n",
      "[episode= 747 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 748 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 749 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 750 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 751 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 752 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 753 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 754 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 755 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 756 ] step = 152\n",
      "updating target-network parmeters...\n",
      "[episode= 757 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 758 ] step = 171\n",
      "updating target-network parmeters...\n",
      "[episode= 759 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 760 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 761 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 762 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 763 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 764 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 765 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 766 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 767 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 768 ] step = 169\n",
      "updating target-network parmeters...\n",
      "[episode= 769 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 770 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 771 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 772 ] step = 158\n",
      "updating target-network parmeters...\n",
      "[episode= 773 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 774 ] step = 154\n",
      "[episode= 775 ] step = 165\n",
      "updating target-network parmeters...\n",
      "[episode= 776 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 777 ] step = 127\n",
      "[episode= 778 ] step = 47\n",
      "updating target-network parmeters...\n",
      "[episode= 779 ] step = 189\n",
      "[episode= 780 ] step = 14\n",
      "updating target-network parmeters...\n",
      "[episode= 781 ] step = 141\n",
      "[episode= 782 ] step = 11\n",
      "updating target-network parmeters...\n",
      "[episode= 783 ] step = 103\n",
      "[episode= 784 ] step = 12\n",
      "[episode= 785 ] step = 11\n",
      "[episode= 786 ] step = 11\n",
      "[episode= 787 ] step = 12\n",
      "[episode= 788 ] step = 12\n",
      "[episode= 789 ] step = 10\n",
      "[episode= 790 ] step = 10\n",
      "[episode= 791 ] step = 10\n",
      "[episode= 792 ] step = 9\n",
      "[episode= 793 ] step = 11\n",
      "[episode= 794 ] step = 10\n",
      "[episode= 795 ] step = 12\n",
      "[episode= 796 ] step = 10\n",
      "[episode= 797 ] step = 13\n",
      "[episode= 798 ] step = 8\n",
      "[episode= 799 ] step = 12\n",
      "updating target-network parmeters...\n",
      "[episode= 800 ] step = 9\n",
      "[episode= 801 ] step = 11\n",
      "[episode= 802 ] step = 10\n",
      "[episode= 803 ] step = 10\n",
      "[episode= 804 ] step = 9\n",
      "[episode= 805 ] step = 10\n",
      "[episode= 806 ] step = 13\n",
      "[episode= 807 ] step = 12\n",
      "[episode= 808 ] step = 11\n",
      "[episode= 809 ] step = 13\n",
      "[episode= 810 ] step = 16\n",
      "[episode= 811 ] step = 14\n",
      "[episode= 812 ] step = 10\n",
      "[episode= 813 ] step = 12\n",
      "[episode= 814 ] step = 9\n",
      "[episode= 815 ] step = 13\n",
      "updating target-network parmeters...\n",
      "[episode= 816 ] step = 13\n",
      "[episode= 817 ] step = 10\n",
      "[episode= 818 ] step = 9\n",
      "[episode= 819 ] step = 12\n",
      "[episode= 820 ] step = 14\n",
      "[episode= 821 ] step = 10\n",
      "[episode= 822 ] step = 10\n",
      "[episode= 823 ] step = 14\n",
      "[episode= 824 ] step = 9\n",
      "[episode= 825 ] step = 11\n",
      "[episode= 826 ] step = 11\n",
      "[episode= 827 ] step = 13\n",
      "[episode= 828 ] step = 11\n",
      "[episode= 829 ] step = 11\n",
      "[episode= 830 ] step = 11\n",
      "[episode= 831 ] step = 13\n",
      "updating target-network parmeters...\n",
      "[episode= 832 ] step = 11\n",
      "[episode= 833 ] step = 10\n",
      "[episode= 834 ] step = 113\n",
      "[episode= 835 ] step = 11\n",
      "[episode= 836 ] step = 13\n",
      "[episode= 837 ] step = 11\n",
      "[episode= 838 ] step = 12\n",
      "[episode= 839 ] step = 12\n",
      "updating target-network parmeters...\n",
      "[episode= 840 ] step = 9\n",
      "[episode= 841 ] step = 11\n",
      "[episode= 842 ] step = 30\n",
      "[episode= 843 ] step = 18\n",
      "[episode= 844 ] step = 25\n",
      "[episode= 845 ] step = 19\n",
      "[episode= 846 ] step = 19\n",
      "[episode= 847 ] step = 45\n",
      "updating target-network parmeters...\n",
      "[episode= 848 ] step = 53\n",
      "[episode= 849 ] step = 90\n",
      "[episode= 850 ] step = 59\n",
      "updating target-network parmeters...\n",
      "[episode= 851 ] step = 98\n",
      "[episode= 852 ] step = 74\n",
      "updating target-network parmeters...\n",
      "[episode= 853 ] step = 131\n",
      "[episode= 854 ] step = 35\n",
      "[episode= 855 ] step = 19\n",
      "updating target-network parmeters...\n",
      "[episode= 856 ] step = 78\n",
      "[episode= 857 ] step = 100\n",
      "[episode= 858 ] step = 73\n",
      "updating target-network parmeters...\n",
      "[episode= 859 ] step = 94\n",
      "[episode= 860 ] step = 101\n",
      "updating target-network parmeters...\n",
      "[episode= 861 ] step = 152\n",
      "updating target-network parmeters...\n",
      "[episode= 862 ] step = 114\n",
      "updating target-network parmeters...\n",
      "[episode= 863 ] step = 171\n",
      "updating target-network parmeters...\n",
      "[episode= 864 ] step = 176\n",
      "[episode= 865 ] step = 130\n",
      "updating target-network parmeters...\n",
      "[episode= 866 ] step = 97\n",
      "[episode= 867 ] step = 79\n",
      "updating target-network parmeters...\n",
      "[episode= 868 ] step = 117\n",
      "[episode= 869 ] step = 159\n",
      "updating target-network parmeters...\n",
      "[episode= 870 ] step = 163\n",
      "updating target-network parmeters...\n",
      "[episode= 871 ] step = 127\n",
      "updating target-network parmeters...\n",
      "[episode= 872 ] step = 126\n",
      "[episode= 873 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode= 874 ] step = 138\n",
      "updating target-network parmeters...\n",
      "[episode= 875 ] step = 174\n",
      "updating target-network parmeters...\n",
      "[episode= 876 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 877 ] step = 192\n",
      "updating target-network parmeters...\n",
      "[episode= 878 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 879 ] step = 199\n",
      "[episode= 880 ] step = 134\n",
      "updating target-network parmeters...\n",
      "[episode= 881 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 882 ] step = 198\n",
      "updating target-network parmeters...\n",
      "[episode= 883 ] step = 155\n",
      "updating target-network parmeters...\n",
      "[episode= 884 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 885 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 886 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 887 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 888 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 889 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 890 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 891 ] step = 199\n",
      "[episode= 892 ] step = 43\n",
      "updating target-network parmeters...\n",
      "[episode= 893 ] step = 135\n",
      "updating target-network parmeters...\n",
      "[episode= 894 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 895 ] step = 72\n",
      "[episode= 896 ] step = 189\n",
      "updating target-network parmeters...\n",
      "[episode= 897 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 898 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 899 ] step = 196\n",
      "updating target-network parmeters...\n",
      "[episode= 900 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 901 ] step = 194\n",
      "updating target-network parmeters...\n",
      "[episode= 902 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 903 ] step = 199\n",
      "updating target-network parmeters...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[episode= 904 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 905 ] step = 193\n",
      "updating target-network parmeters...\n",
      "[episode= 906 ] step = 197\n",
      "updating target-network parmeters...\n",
      "[episode= 907 ] step = 168\n",
      "updating target-network parmeters...\n",
      "[episode= 908 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 909 ] step = 171\n",
      "updating target-network parmeters...\n",
      "[episode= 910 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 911 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 912 ] step = 189\n",
      "updating target-network parmeters...\n",
      "[episode= 913 ] step = 150\n",
      "updating target-network parmeters...\n",
      "[episode= 914 ] step = 153\n",
      "[episode= 915 ] step = 154\n",
      "updating target-network parmeters...\n",
      "[episode= 916 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 917 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 918 ] step = 199\n",
      "[episode= 919 ] step = 30\n",
      "updating target-network parmeters...\n",
      "[episode= 920 ] step = 89\n",
      "updating target-network parmeters...\n",
      "[episode= 921 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 922 ] step = 136\n",
      "[episode= 923 ] step = 160\n",
      "updating target-network parmeters...\n",
      "[episode= 924 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 925 ] step = 26\n",
      "[episode= 926 ] step = 19\n",
      "[episode= 927 ] step = 72\n",
      "updating target-network parmeters...\n",
      "[episode= 928 ] step = 129\n",
      "[episode= 929 ] step = 66\n",
      "[episode= 930 ] step = 24\n",
      "[episode= 931 ] step = 35\n",
      "updating target-network parmeters...\n",
      "[episode= 932 ] step = 151\n",
      "updating target-network parmeters...\n",
      "[episode= 933 ] step = 126\n",
      "[episode= 934 ] step = 140\n",
      "updating target-network parmeters...\n",
      "[episode= 935 ] step = 134\n",
      "updating target-network parmeters...\n",
      "[episode= 936 ] step = 139\n",
      "[episode= 937 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 938 ] step = 108\n",
      "[episode= 939 ] step = 13\n",
      "[episode= 940 ] step = 90\n",
      "updating target-network parmeters...\n",
      "[episode= 941 ] step = 104\n",
      "updating target-network parmeters...\n",
      "[episode= 942 ] step = 101\n",
      "[episode= 943 ] step = 115\n",
      "updating target-network parmeters...\n",
      "[episode= 944 ] step = 130\n",
      "updating target-network parmeters...\n",
      "[episode= 945 ] step = 149\n",
      "[episode= 946 ] step = 134\n",
      "updating target-network parmeters...\n",
      "[episode= 947 ] step = 119\n",
      "updating target-network parmeters...\n",
      "[episode= 948 ] step = 145\n",
      "[episode= 949 ] step = 122\n",
      "updating target-network parmeters...\n",
      "[episode= 950 ] step = 152\n",
      "updating target-network parmeters...\n",
      "[episode= 951 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 952 ] step = 155\n",
      "updating target-network parmeters...\n",
      "[episode= 953 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 954 ] step = 199\n",
      "[episode= 955 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode= 956 ] step = 112\n",
      "updating target-network parmeters...\n",
      "[episode= 957 ] step = 139\n",
      "[episode= 958 ] step = 145\n",
      "updating target-network parmeters...\n",
      "[episode= 959 ] step = 117\n",
      "updating target-network parmeters...\n",
      "[episode= 960 ] step = 123\n",
      "[episode= 961 ] step = 55\n",
      "[episode= 962 ] step = 37\n",
      "[episode= 963 ] step = 83\n",
      "updating target-network parmeters...\n",
      "[episode= 964 ] step = 179\n",
      "updating target-network parmeters...\n",
      "[episode= 965 ] step = 172\n",
      "updating target-network parmeters...\n",
      "[episode= 966 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode= 967 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 968 ] step = 139\n",
      "[episode= 969 ] step = 107\n",
      "updating target-network parmeters...\n",
      "[episode= 970 ] step = 114\n",
      "[episode= 971 ] step = 109\n",
      "updating target-network parmeters...\n",
      "[episode= 972 ] step = 111\n",
      "[episode= 973 ] step = 108\n",
      "updating target-network parmeters...\n",
      "[episode= 974 ] step = 123\n",
      "updating target-network parmeters...\n",
      "[episode= 975 ] step = 157\n",
      "[episode= 976 ] step = 118\n",
      "updating target-network parmeters...\n",
      "[episode= 977 ] step = 118\n",
      "updating target-network parmeters...\n",
      "[episode= 978 ] step = 119\n",
      "[episode= 979 ] step = 120\n",
      "updating target-network parmeters...\n",
      "[episode= 980 ] step = 122\n",
      "[episode= 981 ] step = 116\n",
      "updating target-network parmeters...\n",
      "[episode= 982 ] step = 124\n",
      "updating target-network parmeters...\n",
      "[episode= 983 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 984 ] step = 112\n",
      "updating target-network parmeters...\n",
      "[episode= 985 ] step = 199\n",
      "[episode= 986 ] step = 177\n",
      "updating target-network parmeters...\n",
      "[episode= 987 ] step = 187\n",
      "updating target-network parmeters...\n",
      "[episode= 988 ] step = 161\n",
      "updating target-network parmeters...\n",
      "[episode= 989 ] step = 112\n",
      "updating target-network parmeters...\n",
      "[episode= 990 ] step = 189\n",
      "updating target-network parmeters...\n",
      "[episode= 991 ] step = 199\n",
      "[episode= 992 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 993 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode= 994 ] step = 107\n",
      "[episode= 995 ] step = 113\n",
      "updating target-network parmeters...\n",
      "[episode= 996 ] step = 152\n",
      "updating target-network parmeters...\n",
      "[episode= 997 ] step = 146\n",
      "[episode= 998 ] step = 119\n",
      "updating target-network parmeters...\n",
      "[episode= 999 ] step = 98\n",
      "[episode= 0 ] step = 28\n",
      "[episode= 1 ] step = 16\n",
      "[episode= 2 ] step = 10\n",
      "[episode= 3 ] step = 12\n",
      "[episode= 4 ] step = 18\n",
      "[episode= 5 ] step = 34\n",
      "[episode= 6 ] step = 27\n",
      "[episode= 7 ] step = 11\n",
      "[episode= 8 ] step = 18\n",
      "updating target-network parmeters...\n",
      "[episode= 9 ] step = 18\n",
      "[episode= 10 ] step = 19\n",
      "[episode= 11 ] step = 16\n",
      "[episode= 12 ] step = 13\n",
      "[episode= 13 ] step = 22\n",
      "[episode= 14 ] step = 37\n",
      "[episode= 15 ] step = 17\n",
      "[episode= 16 ] step = 10\n",
      "[episode= 17 ] step = 22\n",
      "updating target-network parmeters...\n",
      "[episode= 18 ] step = 37\n",
      "[episode= 19 ] step = 17\n",
      "[episode= 20 ] step = 23\n",
      "[episode= 21 ] step = 51\n",
      "[episode= 22 ] step = 13\n",
      "[episode= 23 ] step = 34\n",
      "[episode= 24 ] step = 27\n",
      "[episode= 25 ] step = 17\n",
      "updating target-network parmeters...\n",
      "[episode= 26 ] step = 28\n",
      "[episode= 27 ] step = 53\n",
      "[episode= 28 ] step = 31\n",
      "[episode= 29 ] step = 28\n",
      "[episode= 30 ] step = 61\n",
      "updating target-network parmeters...\n",
      "[episode= 31 ] step = 17\n",
      "[episode= 32 ] step = 23\n",
      "[episode= 33 ] step = 27\n",
      "[episode= 34 ] step = 15\n",
      "[episode= 35 ] step = 17\n",
      "[episode= 36 ] step = 23\n",
      "[episode= 37 ] step = 42\n",
      "updating target-network parmeters...\n",
      "[episode= 38 ] step = 40\n",
      "[episode= 39 ] step = 27\n",
      "[episode= 40 ] step = 44\n",
      "[episode= 41 ] step = 35\n",
      "[episode= 42 ] step = 19\n",
      "[episode= 43 ] step = 21\n",
      "[episode= 44 ] step = 22\n",
      "[episode= 45 ] step = 13\n",
      "updating target-network parmeters...\n",
      "[episode= 46 ] step = 91\n",
      "[episode= 47 ] step = 38\n",
      "[episode= 48 ] step = 50\n",
      "updating target-network parmeters...\n",
      "[episode= 49 ] step = 33\n",
      "[episode= 50 ] step = 11\n",
      "[episode= 51 ] step = 54\n",
      "[episode= 52 ] step = 14\n",
      "[episode= 53 ] step = 31\n",
      "[episode= 54 ] step = 65\n",
      "updating target-network parmeters...\n",
      "[episode= 55 ] step = 58\n",
      "[episode= 56 ] step = 28\n",
      "[episode= 57 ] step = 37\n",
      "updating target-network parmeters...\n",
      "[episode= 58 ] step = 119\n",
      "updating target-network parmeters...\n",
      "[episode= 59 ] step = 199\n",
      "[episode= 60 ] step = 80\n",
      "[episode= 61 ] step = 34\n",
      "[episode= 62 ] step = 24\n",
      "updating target-network parmeters...\n",
      "[episode= 63 ] step = 18\n",
      "[episode= 64 ] step = 35\n",
      "[episode= 65 ] step = 98\n",
      "updating target-network parmeters...\n",
      "[episode= 66 ] step = 81\n",
      "[episode= 67 ] step = 41\n",
      "[episode= 68 ] step = 43\n",
      "[episode= 69 ] step = 41\n",
      "[episode= 70 ] step = 27\n",
      "updating target-network parmeters...\n",
      "[episode= 71 ] step = 88\n",
      "[episode= 72 ] step = 111\n",
      "updating target-network parmeters...\n",
      "[episode= 73 ] step = 68\n",
      "updating target-network parmeters...\n",
      "[episode= 74 ] step = 160\n",
      "[episode= 75 ] step = 18\n",
      "[episode= 76 ] step = 34\n",
      "updating target-network parmeters...\n",
      "[episode= 77 ] step = 161\n",
      "[episode= 78 ] step = 162\n",
      "updating target-network parmeters...\n",
      "[episode= 79 ] step = 54\n",
      "[episode= 80 ] step = 101\n",
      "[episode= 81 ] step = 47\n",
      "updating target-network parmeters...\n",
      "[episode= 82 ] step = 195\n",
      "updating target-network parmeters...\n",
      "[episode= 83 ] step = 142\n",
      "updating target-network parmeters...\n",
      "[episode= 84 ] step = 185\n",
      "updating target-network parmeters...\n",
      "[episode= 85 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 86 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 87 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 88 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 89 ] step = 199\n",
      "updating target-network parmeters...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[episode= 90 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 91 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 92 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 93 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 94 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 95 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 96 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 97 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 98 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 99 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 100 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 101 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 102 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 103 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 104 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 105 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 106 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 107 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 108 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 109 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 110 ] step = 190\n",
      "updating target-network parmeters...\n",
      "[episode= 111 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 112 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 113 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 114 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 115 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 116 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 117 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 118 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 119 ] step = 195\n",
      "updating target-network parmeters...\n",
      "[episode= 120 ] step = 193\n",
      "updating target-network parmeters...\n",
      "[episode= 121 ] step = 171\n",
      "updating target-network parmeters...\n",
      "[episode= 122 ] step = 184\n",
      "updating target-network parmeters...\n",
      "[episode= 123 ] step = 195\n",
      "updating target-network parmeters...\n",
      "[episode= 124 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 125 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 126 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 127 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 128 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 129 ] step = 193\n",
      "updating target-network parmeters...\n",
      "[episode= 130 ] step = 199\n",
      "[episode= 131 ] step = 139\n",
      "updating target-network parmeters...\n",
      "[episode= 132 ] step = 91\n",
      "[episode= 133 ] step = 99\n",
      "updating target-network parmeters...\n",
      "[episode= 134 ] step = 89\n",
      "[episode= 135 ] step = 102\n",
      "updating target-network parmeters...\n",
      "[episode= 136 ] step = 130\n",
      "updating target-network parmeters...\n",
      "[episode= 137 ] step = 145\n",
      "updating target-network parmeters...\n",
      "[episode= 138 ] step = 199\n",
      "[episode= 139 ] step = 124\n",
      "updating target-network parmeters...\n",
      "[episode= 140 ] step = 142\n",
      "updating target-network parmeters...\n",
      "[episode= 141 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 142 ] step = 126\n",
      "[episode= 143 ] step = 131\n",
      "updating target-network parmeters...\n",
      "[episode= 144 ] step = 130\n",
      "[episode= 145 ] step = 82\n",
      "updating target-network parmeters...\n",
      "[episode= 146 ] step = 16\n",
      "updating target-network parmeters...\n",
      "[episode= 147 ] step = 199\n",
      "[episode= 148 ] step = 102\n",
      "updating target-network parmeters...\n",
      "[episode= 149 ] step = 104\n",
      "[episode= 150 ] step = 110\n",
      "updating target-network parmeters...\n",
      "[episode= 151 ] step = 106\n",
      "[episode= 152 ] step = 102\n",
      "updating target-network parmeters...\n",
      "[episode= 153 ] step = 104\n",
      "[episode= 154 ] step = 116\n",
      "updating target-network parmeters...\n",
      "[episode= 155 ] step = 114\n",
      "[episode= 156 ] step = 126\n",
      "updating target-network parmeters...\n",
      "[episode= 157 ] step = 83\n",
      "[episode= 158 ] step = 39\n",
      "updating target-network parmeters...\n",
      "[episode= 159 ] step = 95\n",
      "[episode= 160 ] step = 104\n",
      "updating target-network parmeters...\n",
      "[episode= 161 ] step = 106\n",
      "[episode= 162 ] step = 108\n",
      "updating target-network parmeters...\n",
      "[episode= 163 ] step = 97\n",
      "[episode= 164 ] step = 106\n",
      "updating target-network parmeters...\n",
      "[episode= 165 ] step = 66\n",
      "[episode= 166 ] step = 101\n",
      "updating target-network parmeters...\n",
      "[episode= 167 ] step = 93\n",
      "[episode= 168 ] step = 95\n",
      "updating target-network parmeters...\n",
      "[episode= 169 ] step = 155\n",
      "[episode= 170 ] step = 9\n",
      "[episode= 171 ] step = 96\n",
      "updating target-network parmeters...\n",
      "[episode= 172 ] step = 108\n",
      "[episode= 173 ] step = 106\n",
      "updating target-network parmeters...\n",
      "[episode= 174 ] step = 25\n",
      "[episode= 175 ] step = 77\n",
      "[episode= 176 ] step = 43\n",
      "updating target-network parmeters...\n",
      "[episode= 177 ] step = 132\n",
      "[episode= 178 ] step = 70\n",
      "updating target-network parmeters...\n",
      "[episode= 179 ] step = 68\n",
      "[episode= 180 ] step = 93\n",
      "updating target-network parmeters...\n",
      "[episode= 181 ] step = 118\n",
      "[episode= 182 ] step = 106\n",
      "updating target-network parmeters...\n",
      "[episode= 183 ] step = 111\n",
      "[episode= 184 ] step = 102\n",
      "updating target-network parmeters...\n",
      "[episode= 185 ] step = 110\n",
      "[episode= 186 ] step = 101\n",
      "updating target-network parmeters...\n",
      "[episode= 187 ] step = 100\n",
      "[episode= 188 ] step = 116\n",
      "updating target-network parmeters...\n",
      "[episode= 189 ] step = 115\n",
      "[episode= 190 ] step = 102\n",
      "updating target-network parmeters...\n",
      "[episode= 191 ] step = 106\n",
      "updating target-network parmeters...\n",
      "[episode= 192 ] step = 116\n",
      "[episode= 193 ] step = 104\n",
      "updating target-network parmeters...\n",
      "[episode= 194 ] step = 87\n",
      "[episode= 195 ] step = 98\n",
      "updating target-network parmeters...\n",
      "[episode= 196 ] step = 124\n",
      "updating target-network parmeters...\n",
      "[episode= 197 ] step = 199\n",
      "[episode= 198 ] step = 120\n",
      "updating target-network parmeters...\n",
      "[episode= 199 ] step = 120\n",
      "updating target-network parmeters...\n",
      "[episode= 200 ] step = 160\n",
      "[episode= 201 ] step = 114\n",
      "updating target-network parmeters...\n",
      "[episode= 202 ] step = 105\n",
      "[episode= 203 ] step = 97\n",
      "updating target-network parmeters...\n",
      "[episode= 204 ] step = 97\n",
      "[episode= 205 ] step = 102\n",
      "updating target-network parmeters...\n",
      "[episode= 206 ] step = 118\n",
      "[episode= 207 ] step = 80\n",
      "updating target-network parmeters...\n",
      "[episode= 208 ] step = 141\n",
      "updating target-network parmeters...\n",
      "[episode= 209 ] step = 127\n",
      "[episode= 210 ] step = 123\n",
      "updating target-network parmeters...\n",
      "[episode= 211 ] step = 121\n",
      "[episode= 212 ] step = 127\n",
      "updating target-network parmeters...\n",
      "[episode= 213 ] step = 145\n",
      "updating target-network parmeters...\n",
      "[episode= 214 ] step = 141\n",
      "updating target-network parmeters...\n",
      "[episode= 215 ] step = 126\n",
      "[episode= 216 ] step = 153\n",
      "updating target-network parmeters...\n",
      "[episode= 217 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode= 218 ] step = 150\n",
      "updating target-network parmeters...\n",
      "[episode= 219 ] step = 157\n",
      "[episode= 220 ] step = 131\n",
      "updating target-network parmeters...\n",
      "[episode= 221 ] step = 165\n",
      "updating target-network parmeters...\n",
      "[episode= 222 ] step = 170\n",
      "updating target-network parmeters...\n",
      "[episode= 223 ] step = 141\n",
      "updating target-network parmeters...\n",
      "[episode= 224 ] step = 180\n",
      "[episode= 225 ] step = 141\n",
      "updating target-network parmeters...\n",
      "[episode= 226 ] step = 147\n",
      "updating target-network parmeters...\n",
      "[episode= 227 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode= 228 ] step = 199\n",
      "[episode= 229 ] step = 148\n",
      "updating target-network parmeters...\n",
      "[episode= 230 ] step = 157\n",
      "updating target-network parmeters...\n",
      "[episode= 231 ] step = 161\n",
      "updating target-network parmeters...\n",
      "[episode= 232 ] step = 125\n",
      "[episode= 233 ] step = 120\n",
      "updating target-network parmeters...\n",
      "[episode= 234 ] step = 129\n",
      "updating target-network parmeters...\n",
      "[episode= 235 ] step = 193\n",
      "[episode= 236 ] step = 120\n",
      "updating target-network parmeters...\n",
      "[episode= 237 ] step = 114\n",
      "updating target-network parmeters...\n",
      "[episode= 238 ] step = 117\n",
      "[episode= 239 ] step = 139\n",
      "updating target-network parmeters...\n",
      "[episode= 240 ] step = 115\n",
      "[episode= 241 ] step = 112\n",
      "updating target-network parmeters...\n",
      "[episode= 242 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 243 ] step = 118\n",
      "[episode= 244 ] step = 109\n",
      "updating target-network parmeters...\n",
      "[episode= 245 ] step = 102\n",
      "[episode= 246 ] step = 14\n",
      "[episode= 247 ] step = 10\n",
      "[episode= 248 ] step = 15\n",
      "[episode= 249 ] step = 12\n",
      "[episode= 250 ] step = 14\n",
      "[episode= 251 ] step = 13\n",
      "[episode= 252 ] step = 13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[episode= 253 ] step = 26\n",
      "updating target-network parmeters...\n",
      "[episode= 254 ] step = 94\n",
      "[episode= 255 ] step = 116\n",
      "updating target-network parmeters...\n",
      "[episode= 256 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 257 ] step = 98\n",
      "[episode= 258 ] step = 109\n",
      "updating target-network parmeters...\n",
      "[episode= 259 ] step = 118\n",
      "[episode= 260 ] step = 108\n",
      "updating target-network parmeters...\n",
      "[episode= 261 ] step = 121\n",
      "[episode= 262 ] step = 133\n",
      "updating target-network parmeters...\n",
      "[episode= 263 ] step = 116\n",
      "updating target-network parmeters...\n",
      "[episode= 264 ] step = 126\n",
      "updating target-network parmeters...\n",
      "[episode= 265 ] step = 167\n",
      "updating target-network parmeters...\n",
      "[episode= 266 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 267 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 268 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 269 ] step = 199\n",
      "[episode= 270 ] step = 170\n",
      "updating target-network parmeters...\n",
      "[episode= 271 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 272 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 273 ] step = 149\n",
      "updating target-network parmeters...\n",
      "[episode= 274 ] step = 158\n",
      "[episode= 275 ] step = 110\n",
      "updating target-network parmeters...\n",
      "[episode= 276 ] step = 84\n",
      "[episode= 277 ] step = 101\n",
      "updating target-network parmeters...\n",
      "[episode= 278 ] step = 70\n",
      "[episode= 279 ] step = 69\n",
      "[episode= 280 ] step = 9\n",
      "updating target-network parmeters...\n",
      "[episode= 281 ] step = 90\n",
      "[episode= 282 ] step = 58\n",
      "[episode= 283 ] step = 81\n",
      "updating target-network parmeters...\n",
      "[episode= 284 ] step = 46\n",
      "[episode= 285 ] step = 27\n",
      "[episode= 286 ] step = 33\n",
      "[episode= 287 ] step = 11\n",
      "[episode= 288 ] step = 10\n",
      "[episode= 289 ] step = 11\n",
      "[episode= 290 ] step = 27\n",
      "[episode= 291 ] step = 30\n",
      "[episode= 292 ] step = 9\n",
      "[episode= 293 ] step = 10\n",
      "updating target-network parmeters...\n",
      "[episode= 294 ] step = 8\n",
      "[episode= 295 ] step = 10\n",
      "[episode= 296 ] step = 52\n",
      "[episode= 297 ] step = 10\n",
      "[episode= 298 ] step = 54\n",
      "[episode= 299 ] step = 11\n",
      "[episode= 300 ] step = 12\n",
      "updating target-network parmeters...\n",
      "[episode= 301 ] step = 146\n",
      "[episode= 302 ] step = 12\n",
      "[episode= 303 ] step = 13\n",
      "updating target-network parmeters...\n",
      "[episode= 304 ] step = 134\n",
      "updating target-network parmeters...\n",
      "[episode= 305 ] step = 199\n",
      "[episode= 306 ] step = 115\n",
      "updating target-network parmeters...\n",
      "[episode= 307 ] step = 165\n",
      "updating target-network parmeters...\n",
      "[episode= 308 ] step = 139\n",
      "updating target-network parmeters...\n",
      "[episode= 309 ] step = 183\n",
      "updating target-network parmeters...\n",
      "[episode= 310 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 311 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 312 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 313 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 314 ] step = 171\n",
      "[episode= 315 ] step = 129\n",
      "updating target-network parmeters...\n",
      "[episode= 316 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 317 ] step = 127\n",
      "updating target-network parmeters...\n",
      "[episode= 318 ] step = 120\n",
      "[episode= 319 ] step = 134\n",
      "updating target-network parmeters...\n",
      "[episode= 320 ] step = 116\n",
      "[episode= 321 ] step = 92\n",
      "updating target-network parmeters...\n",
      "[episode= 322 ] step = 111\n",
      "updating target-network parmeters...\n",
      "[episode= 323 ] step = 142\n",
      "[episode= 324 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 325 ] step = 139\n",
      "[episode= 326 ] step = 99\n",
      "updating target-network parmeters...\n",
      "[episode= 327 ] step = 171\n",
      "updating target-network parmeters...\n",
      "[episode= 328 ] step = 161\n",
      "updating target-network parmeters...\n",
      "[episode= 329 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode= 330 ] step = 147\n",
      "[episode= 331 ] step = 99\n",
      "updating target-network parmeters...\n",
      "[episode= 332 ] step = 98\n",
      "[episode= 333 ] step = 58\n",
      "[episode= 334 ] step = 64\n",
      "updating target-network parmeters...\n",
      "[episode= 335 ] step = 114\n",
      "[episode= 336 ] step = 71\n",
      "updating target-network parmeters...\n",
      "[episode= 337 ] step = 112\n",
      "[episode= 338 ] step = 157\n",
      "updating target-network parmeters...\n",
      "[episode= 339 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 340 ] step = 90\n",
      "updating target-network parmeters...\n",
      "[episode= 341 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 342 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 343 ] step = 148\n",
      "[episode= 344 ] step = 85\n",
      "updating target-network parmeters...\n",
      "[episode= 345 ] step = 117\n",
      "[episode= 346 ] step = 146\n",
      "updating target-network parmeters...\n",
      "[episode= 347 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 348 ] step = 179\n",
      "updating target-network parmeters...\n",
      "[episode= 349 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 350 ] step = 180\n",
      "updating target-network parmeters...\n",
      "[episode= 351 ] step = 155\n",
      "updating target-network parmeters...\n",
      "[episode= 352 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 353 ] step = 147\n",
      "updating target-network parmeters...\n",
      "[episode= 354 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 355 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 356 ] step = 175\n",
      "[episode= 357 ] step = 149\n",
      "updating target-network parmeters...\n",
      "[episode= 358 ] step = 143\n",
      "updating target-network parmeters...\n",
      "[episode= 359 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 360 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 361 ] step = 113\n",
      "[episode= 362 ] step = 173\n",
      "updating target-network parmeters...\n",
      "[episode= 363 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 364 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 365 ] step = 184\n",
      "updating target-network parmeters...\n",
      "[episode= 366 ] step = 115\n",
      "updating target-network parmeters...\n",
      "[episode= 367 ] step = 103\n",
      "[episode= 368 ] step = 103\n",
      "updating target-network parmeters...\n",
      "[episode= 369 ] step = 136\n",
      "[episode= 370 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 371 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 372 ] step = 155\n",
      "updating target-network parmeters...\n",
      "[episode= 373 ] step = 97\n",
      "[episode= 374 ] step = 103\n",
      "updating target-network parmeters...\n",
      "[episode= 375 ] step = 116\n",
      "[episode= 376 ] step = 107\n",
      "updating target-network parmeters...\n",
      "[episode= 377 ] step = 111\n",
      "updating target-network parmeters...\n",
      "[episode= 378 ] step = 187\n",
      "[episode= 379 ] step = 101\n",
      "[episode= 380 ] step = 16\n",
      "[episode= 381 ] step = 15\n",
      "updating target-network parmeters...\n",
      "[episode= 382 ] step = 15\n",
      "[episode= 383 ] step = 21\n",
      "[episode= 384 ] step = 14\n",
      "[episode= 385 ] step = 108\n",
      "updating target-network parmeters...\n",
      "[episode= 386 ] step = 106\n",
      "[episode= 387 ] step = 114\n",
      "updating target-network parmeters...\n",
      "[episode= 388 ] step = 164\n",
      "updating target-network parmeters...\n",
      "[episode= 389 ] step = 154\n",
      "updating target-network parmeters...\n",
      "[episode= 390 ] step = 138\n",
      "[episode= 391 ] step = 163\n",
      "updating target-network parmeters...\n",
      "[episode= 392 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 393 ] step = 103\n",
      "updating target-network parmeters...\n",
      "[episode= 394 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 395 ] step = 174\n",
      "[episode= 396 ] step = 13\n",
      "updating target-network parmeters...\n",
      "[episode= 397 ] step = 156\n",
      "[episode= 398 ] step = 11\n",
      "[episode= 399 ] step = 86\n",
      "[episode= 400 ] step = 10\n",
      "[episode= 401 ] step = 41\n",
      "updating target-network parmeters...\n",
      "[episode= 402 ] step = 16\n",
      "[episode= 403 ] step = 15\n",
      "[episode= 404 ] step = 162\n",
      "updating target-network parmeters...\n",
      "[episode= 405 ] step = 137\n",
      "updating target-network parmeters...\n",
      "[episode= 406 ] step = 104\n",
      "[episode= 407 ] step = 132\n",
      "updating target-network parmeters...\n",
      "[episode= 408 ] step = 97\n",
      "[episode= 409 ] step = 132\n",
      "updating target-network parmeters...\n",
      "[episode= 410 ] step = 92\n",
      "[episode= 411 ] step = 108\n",
      "updating target-network parmeters...\n",
      "[episode= 412 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 413 ] step = 126\n",
      "updating target-network parmeters...\n",
      "[episode= 414 ] step = 199\n",
      "[episode= 415 ] step = 19\n",
      "updating target-network parmeters...\n",
      "[episode= 416 ] step = 131\n",
      "updating target-network parmeters...\n",
      "[episode= 417 ] step = 141\n",
      "[episode= 418 ] step = 10\n",
      "[episode= 419 ] step = 29\n",
      "[episode= 420 ] step = 27\n",
      "[episode= 421 ] step = 27\n",
      "updating target-network parmeters...\n",
      "[episode= 422 ] step = 197\n",
      "updating target-network parmeters...\n",
      "[episode= 423 ] step = 120\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "updating target-network parmeters...\n",
      "[episode= 424 ] step = 199\n",
      "[episode= 425 ] step = 139\n",
      "updating target-network parmeters...\n",
      "[episode= 426 ] step = 168\n",
      "updating target-network parmeters...\n",
      "[episode= 427 ] step = 137\n",
      "updating target-network parmeters...\n",
      "[episode= 428 ] step = 123\n",
      "[episode= 429 ] step = 117\n",
      "updating target-network parmeters...\n",
      "[episode= 430 ] step = 165\n",
      "updating target-network parmeters...\n",
      "[episode= 431 ] step = 120\n",
      "[episode= 432 ] step = 116\n",
      "updating target-network parmeters...\n",
      "[episode= 433 ] step = 108\n",
      "[episode= 434 ] step = 150\n",
      "updating target-network parmeters...\n",
      "[episode= 435 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 436 ] step = 100\n",
      "[episode= 437 ] step = 88\n",
      "updating target-network parmeters...\n",
      "[episode= 438 ] step = 196\n",
      "[episode= 439 ] step = 98\n",
      "updating target-network parmeters...\n",
      "[episode= 440 ] step = 127\n",
      "updating target-network parmeters...\n",
      "[episode= 441 ] step = 95\n",
      "[episode= 442 ] step = 66\n",
      "[episode= 443 ] step = 93\n",
      "updating target-network parmeters...\n",
      "[episode= 444 ] step = 119\n",
      "updating target-network parmeters...\n",
      "[episode= 445 ] step = 144\n",
      "[episode= 446 ] step = 132\n",
      "updating target-network parmeters...\n",
      "[episode= 447 ] step = 111\n",
      "[episode= 448 ] step = 103\n",
      "updating target-network parmeters...\n",
      "[episode= 449 ] step = 123\n",
      "updating target-network parmeters...\n",
      "[episode= 450 ] step = 155\n",
      "updating target-network parmeters...\n",
      "[episode= 451 ] step = 138\n",
      "[episode= 452 ] step = 135\n",
      "updating target-network parmeters...\n",
      "[episode= 453 ] step = 116\n",
      "updating target-network parmeters...\n",
      "[episode= 454 ] step = 132\n",
      "[episode= 455 ] step = 117\n",
      "updating target-network parmeters...\n",
      "[episode= 456 ] step = 142\n",
      "updating target-network parmeters...\n",
      "[episode= 457 ] step = 147\n",
      "[episode= 458 ] step = 161\n",
      "updating target-network parmeters...\n",
      "[episode= 459 ] step = 150\n",
      "updating target-network parmeters...\n",
      "[episode= 460 ] step = 170\n",
      "updating target-network parmeters...\n",
      "[episode= 461 ] step = 155\n",
      "updating target-network parmeters...\n",
      "[episode= 462 ] step = 158\n",
      "[episode= 463 ] step = 145\n",
      "updating target-network parmeters...\n",
      "[episode= 464 ] step = 133\n",
      "updating target-network parmeters...\n",
      "[episode= 465 ] step = 166\n",
      "updating target-network parmeters...\n",
      "[episode= 466 ] step = 199\n",
      "[episode= 467 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode= 468 ] step = 128\n",
      "updating target-network parmeters...\n",
      "[episode= 469 ] step = 163\n",
      "[episode= 470 ] step = 101\n",
      "updating target-network parmeters...\n",
      "[episode= 471 ] step = 139\n",
      "updating target-network parmeters...\n",
      "[episode= 472 ] step = 135\n",
      "[episode= 473 ] step = 97\n",
      "updating target-network parmeters...\n",
      "[episode= 474 ] step = 102\n",
      "updating target-network parmeters...\n",
      "[episode= 475 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 476 ] step = 154\n",
      "[episode= 477 ] step = 128\n",
      "updating target-network parmeters...\n",
      "[episode= 478 ] step = 132\n",
      "[episode= 479 ] step = 96\n",
      "updating target-network parmeters...\n",
      "[episode= 480 ] step = 178\n",
      "updating target-network parmeters...\n",
      "[episode= 481 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 482 ] step = 128\n",
      "[episode= 483 ] step = 107\n",
      "updating target-network parmeters...\n",
      "[episode= 484 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 485 ] step = 198\n",
      "updating target-network parmeters...\n",
      "[episode= 486 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 487 ] step = 67\n",
      "[episode= 488 ] step = 66\n",
      "updating target-network parmeters...\n",
      "[episode= 489 ] step = 178\n",
      "updating target-network parmeters...\n",
      "[episode= 490 ] step = 173\n",
      "updating target-network parmeters...\n",
      "[episode= 491 ] step = 140\n",
      "[episode= 492 ] step = 135\n",
      "updating target-network parmeters...\n",
      "[episode= 493 ] step = 105\n",
      "[episode= 494 ] step = 110\n",
      "updating target-network parmeters...\n",
      "[episode= 495 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 496 ] step = 142\n",
      "updating target-network parmeters...\n",
      "[episode= 497 ] step = 199\n",
      "[episode= 498 ] step = 153\n",
      "updating target-network parmeters...\n",
      "[episode= 499 ] step = 99\n",
      "updating target-network parmeters...\n",
      "[episode= 500 ] step = 118\n",
      "updating target-network parmeters...\n",
      "[episode= 501 ] step = 199\n",
      "[episode= 502 ] step = 126\n",
      "updating target-network parmeters...\n",
      "[episode= 503 ] step = 167\n",
      "updating target-network parmeters...\n",
      "[episode= 504 ] step = 102\n",
      "[episode= 505 ] step = 98\n",
      "updating target-network parmeters...\n",
      "[episode= 506 ] step = 101\n",
      "[episode= 507 ] step = 175\n",
      "updating target-network parmeters...\n",
      "[episode= 508 ] step = 154\n",
      "updating target-network parmeters...\n",
      "[episode= 509 ] step = 146\n",
      "[episode= 510 ] step = 104\n",
      "updating target-network parmeters...\n",
      "[episode= 511 ] step = 114\n",
      "updating target-network parmeters...\n",
      "[episode= 512 ] step = 137\n",
      "updating target-network parmeters...\n",
      "[episode= 513 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 514 ] step = 177\n",
      "updating target-network parmeters...\n",
      "[episode= 515 ] step = 199\n",
      "[episode= 516 ] step = 162\n",
      "updating target-network parmeters...\n",
      "[episode= 517 ] step = 71\n",
      "[episode= 518 ] step = 92\n",
      "updating target-network parmeters...\n",
      "[episode= 519 ] step = 124\n",
      "[episode= 520 ] step = 13\n",
      "[episode= 521 ] step = 96\n",
      "updating target-network parmeters...\n",
      "[episode= 522 ] step = 101\n",
      "updating target-network parmeters...\n",
      "[episode= 523 ] step = 123\n",
      "[episode= 524 ] step = 110\n",
      "updating target-network parmeters...\n",
      "[episode= 525 ] step = 98\n",
      "[episode= 526 ] step = 89\n",
      "[episode= 527 ] step = 44\n",
      "[episode= 528 ] step = 15\n",
      "[episode= 529 ] step = 18\n",
      "updating target-network parmeters...\n",
      "[episode= 530 ] step = 44\n",
      "[episode= 531 ] step = 49\n",
      "updating target-network parmeters...\n",
      "[episode= 532 ] step = 138\n",
      "[episode= 533 ] step = 116\n",
      "[episode= 534 ] step = 25\n",
      "updating target-network parmeters...\n",
      "[episode= 535 ] step = 95\n",
      "[episode= 536 ] step = 118\n",
      "updating target-network parmeters...\n",
      "[episode= 537 ] step = 138\n",
      "[episode= 538 ] step = 31\n",
      "updating target-network parmeters...\n",
      "[episode= 539 ] step = 62\n",
      "[episode= 540 ] step = 59\n",
      "[episode= 541 ] step = 21\n",
      "updating target-network parmeters...\n",
      "[episode= 542 ] step = 199\n",
      "[episode= 543 ] step = 25\n",
      "updating target-network parmeters...\n",
      "[episode= 544 ] step = 110\n",
      "updating target-network parmeters...\n",
      "[episode= 545 ] step = 182\n",
      "[episode= 546 ] step = 149\n",
      "updating target-network parmeters...\n",
      "[episode= 547 ] step = 148\n",
      "updating target-network parmeters...\n",
      "[episode= 548 ] step = 194\n",
      "updating target-network parmeters...\n",
      "[episode= 549 ] step = 139\n",
      "[episode= 550 ] step = 20\n",
      "[episode= 551 ] step = 84\n",
      "updating target-network parmeters...\n",
      "[episode= 552 ] step = 87\n",
      "[episode= 553 ] step = 103\n",
      "updating target-network parmeters...\n",
      "[episode= 554 ] step = 128\n",
      "updating target-network parmeters...\n",
      "[episode= 555 ] step = 125\n",
      "[episode= 556 ] step = 120\n",
      "updating target-network parmeters...\n",
      "[episode= 557 ] step = 139\n",
      "[episode= 558 ] step = 107\n",
      "updating target-network parmeters...\n",
      "[episode= 559 ] step = 134\n",
      "updating target-network parmeters...\n",
      "[episode= 560 ] step = 140\n",
      "updating target-network parmeters...\n",
      "[episode= 561 ] step = 149\n",
      "[episode= 562 ] step = 179\n",
      "updating target-network parmeters...\n",
      "[episode= 563 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 564 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 565 ] step = 175\n",
      "updating target-network parmeters...\n",
      "[episode= 566 ] step = 123\n",
      "[episode= 567 ] step = 110\n",
      "updating target-network parmeters...\n",
      "[episode= 568 ] step = 120\n",
      "updating target-network parmeters...\n",
      "[episode= 569 ] step = 150\n",
      "[episode= 570 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 571 ] step = 176\n",
      "updating target-network parmeters...\n",
      "[episode= 572 ] step = 129\n",
      "updating target-network parmeters...\n",
      "[episode= 573 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 574 ] step = 105\n",
      "[episode= 575 ] step = 157\n",
      "updating target-network parmeters...\n",
      "[episode= 576 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 577 ] step = 132\n",
      "updating target-network parmeters...\n",
      "[episode= 578 ] step = 120\n",
      "[episode= 579 ] step = 82\n",
      "updating target-network parmeters...\n",
      "[episode= 580 ] step = 166\n",
      "[episode= 581 ] step = 88\n",
      "updating target-network parmeters...\n",
      "[episode= 582 ] step = 160\n",
      "updating target-network parmeters...\n",
      "[episode= 583 ] step = 199\n",
      "[episode= 584 ] step = 26\n",
      "updating target-network parmeters...\n",
      "[episode= 585 ] step = 199\n",
      "[episode= 586 ] step = 41\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "updating target-network parmeters...\n",
      "[episode= 587 ] step = 187\n",
      "updating target-network parmeters...\n",
      "[episode= 588 ] step = 158\n",
      "updating target-network parmeters...\n",
      "[episode= 589 ] step = 89\n",
      "[episode= 590 ] step = 118\n",
      "updating target-network parmeters...\n",
      "[episode= 591 ] step = 140\n",
      "updating target-network parmeters...\n",
      "[episode= 592 ] step = 171\n",
      "updating target-network parmeters...\n",
      "[episode= 593 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 594 ] step = 144\n",
      "updating target-network parmeters...\n",
      "[episode= 595 ] step = 199\n",
      "[episode= 596 ] step = 123\n",
      "updating target-network parmeters...\n",
      "[episode= 597 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 598 ] step = 156\n",
      "updating target-network parmeters...\n",
      "[episode= 599 ] step = 145\n",
      "[episode= 600 ] step = 113\n",
      "updating target-network parmeters...\n",
      "[episode= 601 ] step = 137\n",
      "updating target-network parmeters...\n",
      "[episode= 602 ] step = 168\n",
      "updating target-network parmeters...\n",
      "[episode= 603 ] step = 166\n",
      "[episode= 604 ] step = 149\n",
      "updating target-network parmeters...\n",
      "[episode= 605 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 606 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 607 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 608 ] step = 175\n",
      "updating target-network parmeters...\n",
      "[episode= 609 ] step = 159\n",
      "updating target-network parmeters...\n",
      "[episode= 610 ] step = 143\n",
      "updating target-network parmeters...\n",
      "[episode= 611 ] step = 164\n",
      "[episode= 612 ] step = 183\n",
      "updating target-network parmeters...\n",
      "[episode= 613 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 614 ] step = 145\n",
      "updating target-network parmeters...\n",
      "[episode= 615 ] step = 160\n",
      "updating target-network parmeters...\n",
      "[episode= 616 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 617 ] step = 132\n",
      "[episode= 618 ] step = 139\n",
      "updating target-network parmeters...\n",
      "[episode= 619 ] step = 141\n",
      "[episode= 620 ] step = 30\n",
      "updating target-network parmeters...\n",
      "[episode= 621 ] step = 127\n",
      "updating target-network parmeters...\n",
      "[episode= 622 ] step = 157\n",
      "[episode= 623 ] step = 132\n",
      "[episode= 624 ] step = 15\n",
      "[episode= 625 ] step = 13\n",
      "updating target-network parmeters...\n",
      "[episode= 626 ] step = 124\n",
      "updating target-network parmeters...\n",
      "[episode= 627 ] step = 152\n",
      "updating target-network parmeters...\n",
      "[episode= 628 ] step = 199\n",
      "[episode= 629 ] step = 102\n",
      "updating target-network parmeters...\n",
      "[episode= 630 ] step = 175\n",
      "updating target-network parmeters...\n",
      "[episode= 631 ] step = 91\n",
      "[episode= 632 ] step = 144\n",
      "updating target-network parmeters...\n",
      "[episode= 633 ] step = 195\n",
      "updating target-network parmeters...\n",
      "[episode= 634 ] step = 88\n",
      "[episode= 635 ] step = 105\n",
      "updating target-network parmeters...\n",
      "[episode= 636 ] step = 108\n",
      "updating target-network parmeters...\n",
      "[episode= 637 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 638 ] step = 149\n",
      "updating target-network parmeters...\n",
      "[episode= 639 ] step = 168\n",
      "updating target-network parmeters...\n",
      "[episode= 640 ] step = 194\n",
      "updating target-network parmeters...\n",
      "[episode= 641 ] step = 199\n",
      "[episode= 642 ] step = 134\n",
      "updating target-network parmeters...\n",
      "[episode= 643 ] step = 137\n",
      "updating target-network parmeters...\n",
      "[episode= 644 ] step = 173\n",
      "[episode= 645 ] step = 94\n",
      "updating target-network parmeters...\n",
      "[episode= 646 ] step = 113\n",
      "[episode= 647 ] step = 98\n",
      "updating target-network parmeters...\n",
      "[episode= 648 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 649 ] step = 132\n",
      "[episode= 650 ] step = 38\n",
      "updating target-network parmeters...\n",
      "[episode= 651 ] step = 175\n",
      "updating target-network parmeters...\n",
      "[episode= 652 ] step = 142\n",
      "[episode= 653 ] step = 133\n",
      "updating target-network parmeters...\n",
      "[episode= 654 ] step = 144\n",
      "updating target-network parmeters...\n",
      "[episode= 655 ] step = 119\n",
      "[episode= 656 ] step = 145\n",
      "updating target-network parmeters...\n",
      "[episode= 657 ] step = 146\n",
      "updating target-network parmeters...\n",
      "[episode= 658 ] step = 142\n",
      "updating target-network parmeters...\n",
      "[episode= 659 ] step = 145\n",
      "[episode= 660 ] step = 144\n",
      "updating target-network parmeters...\n",
      "[episode= 661 ] step = 145\n",
      "updating target-network parmeters...\n",
      "[episode= 662 ] step = 156\n",
      "updating target-network parmeters...\n",
      "[episode= 663 ] step = 147\n",
      "[episode= 664 ] step = 138\n",
      "updating target-network parmeters...\n",
      "[episode= 665 ] step = 130\n",
      "updating target-network parmeters...\n",
      "[episode= 666 ] step = 140\n",
      "updating target-network parmeters...\n",
      "[episode= 667 ] step = 163\n",
      "[episode= 668 ] step = 125\n",
      "updating target-network parmeters...\n",
      "[episode= 669 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 670 ] step = 117\n",
      "updating target-network parmeters...\n",
      "[episode= 671 ] step = 169\n",
      "[episode= 672 ] step = 46\n",
      "updating target-network parmeters...\n",
      "[episode= 673 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 674 ] step = 156\n",
      "updating target-network parmeters...\n",
      "[episode= 675 ] step = 199\n",
      "[episode= 676 ] step = 151\n",
      "updating target-network parmeters...\n",
      "[episode= 677 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 678 ] step = 138\n",
      "updating target-network parmeters...\n",
      "[episode= 679 ] step = 124\n",
      "[episode= 680 ] step = 146\n",
      "updating target-network parmeters...\n",
      "[episode= 681 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode= 682 ] step = 90\n",
      "[episode= 683 ] step = 72\n",
      "updating target-network parmeters...\n",
      "[episode= 684 ] step = 126\n",
      "[episode= 685 ] step = 112\n",
      "updating target-network parmeters...\n",
      "[episode= 686 ] step = 135\n",
      "[episode= 687 ] step = 95\n",
      "updating target-network parmeters...\n",
      "[episode= 688 ] step = 115\n",
      "[episode= 689 ] step = 114\n",
      "updating target-network parmeters...\n",
      "[episode= 690 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 691 ] step = 127\n",
      "[episode= 692 ] step = 110\n",
      "updating target-network parmeters...\n",
      "[episode= 693 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 694 ] step = 162\n",
      "updating target-network parmeters...\n",
      "[episode= 695 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 696 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 697 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 698 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 699 ] step = 175\n",
      "updating target-network parmeters...\n",
      "[episode= 700 ] step = 115\n",
      "[episode= 701 ] step = 110\n",
      "updating target-network parmeters...\n",
      "[episode= 702 ] step = 196\n",
      "updating target-network parmeters...\n",
      "[episode= 703 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 704 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 705 ] step = 157\n",
      "[episode= 706 ] step = 68\n",
      "updating target-network parmeters...\n",
      "[episode= 707 ] step = 110\n",
      "[episode= 708 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 709 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 710 ] step = 34\n",
      "updating target-network parmeters...\n",
      "[episode= 711 ] step = 199\n",
      "[episode= 712 ] step = 45\n",
      "[episode= 713 ] step = 17\n",
      "[episode= 714 ] step = 44\n",
      "updating target-network parmeters...\n",
      "[episode= 715 ] step = 156\n",
      "updating target-network parmeters...\n",
      "[episode= 716 ] step = 183\n",
      "updating target-network parmeters...\n",
      "[episode= 717 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 718 ] step = 192\n",
      "[episode= 719 ] step = 137\n",
      "updating target-network parmeters...\n",
      "[episode= 720 ] step = 36\n",
      "updating target-network parmeters...\n",
      "[episode= 721 ] step = 183\n",
      "[episode= 722 ] step = 162\n",
      "updating target-network parmeters...\n",
      "[episode= 723 ] step = 134\n",
      "[episode= 724 ] step = 91\n",
      "updating target-network parmeters...\n",
      "[episode= 725 ] step = 47\n",
      "[episode= 726 ] step = 134\n",
      "updating target-network parmeters...\n",
      "[episode= 727 ] step = 106\n",
      "[episode= 728 ] step = 52\n",
      "updating target-network parmeters...\n",
      "[episode= 729 ] step = 119\n",
      "[episode= 730 ] step = 88\n",
      "updating target-network parmeters...\n",
      "[episode= 731 ] step = 51\n",
      "[episode= 732 ] step = 32\n",
      "[episode= 733 ] step = 163\n",
      "updating target-network parmeters...\n",
      "[episode= 734 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 735 ] step = 78\n",
      "[episode= 736 ] step = 109\n",
      "updating target-network parmeters...\n",
      "[episode= 737 ] step = 148\n",
      "updating target-network parmeters...\n",
      "[episode= 738 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 739 ] step = 140\n",
      "[episode= 740 ] step = 61\n",
      "updating target-network parmeters...\n",
      "[episode= 741 ] step = 106\n",
      "updating target-network parmeters...\n",
      "[episode= 742 ] step = 159\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[episode= 743 ] step = 138\n",
      "[episode= 744 ] step = 51\n",
      "updating target-network parmeters...\n",
      "[episode= 745 ] step = 115\n",
      "[episode= 746 ] step = 59\n",
      "updating target-network parmeters...\n",
      "[episode= 747 ] step = 84\n",
      "[episode= 748 ] step = 29\n",
      "[episode= 749 ] step = 28\n",
      "[episode= 750 ] step = 28\n",
      "[episode= 751 ] step = 29\n",
      "updating target-network parmeters...\n",
      "[episode= 752 ] step = 27\n",
      "[episode= 753 ] step = 27\n",
      "[episode= 754 ] step = 25\n",
      "[episode= 755 ] step = 20\n",
      "[episode= 756 ] step = 88\n",
      "[episode= 757 ] step = 22\n",
      "updating target-network parmeters...\n",
      "[episode= 758 ] step = 118\n",
      "[episode= 759 ] step = 11\n",
      "updating target-network parmeters...\n",
      "[episode= 760 ] step = 121\n",
      "[episode= 761 ] step = 125\n",
      "updating target-network parmeters...\n",
      "[episode= 762 ] step = 49\n",
      "updating target-network parmeters...\n",
      "[episode= 763 ] step = 183\n",
      "[episode= 764 ] step = 125\n",
      "updating target-network parmeters...\n",
      "[episode= 765 ] step = 91\n",
      "[episode= 766 ] step = 100\n",
      "updating target-network parmeters...\n",
      "[episode= 767 ] step = 110\n",
      "updating target-network parmeters...\n",
      "[episode= 768 ] step = 199\n",
      "[episode= 769 ] step = 101\n",
      "updating target-network parmeters...\n",
      "[episode= 770 ] step = 103\n",
      "[episode= 771 ] step = 42\n",
      "updating target-network parmeters...\n",
      "[episode= 772 ] step = 158\n",
      "[episode= 773 ] step = 145\n",
      "updating target-network parmeters...\n",
      "[episode= 774 ] step = 142\n",
      "updating target-network parmeters...\n",
      "[episode= 775 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 776 ] step = 123\n",
      "[episode= 777 ] step = 99\n",
      "updating target-network parmeters...\n",
      "[episode= 778 ] step = 168\n",
      "updating target-network parmeters...\n",
      "[episode= 779 ] step = 120\n",
      "[episode= 780 ] step = 132\n",
      "updating target-network parmeters...\n",
      "[episode= 781 ] step = 104\n",
      "[episode= 782 ] step = 108\n",
      "updating target-network parmeters...\n",
      "[episode= 783 ] step = 108\n",
      "updating target-network parmeters...\n",
      "[episode= 784 ] step = 131\n",
      "[episode= 785 ] step = 114\n",
      "updating target-network parmeters...\n",
      "[episode= 786 ] step = 114\n",
      "updating target-network parmeters...\n",
      "[episode= 787 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 788 ] step = 138\n",
      "[episode= 789 ] step = 128\n",
      "updating target-network parmeters...\n",
      "[episode= 790 ] step = 154\n",
      "updating target-network parmeters...\n",
      "[episode= 791 ] step = 138\n",
      "[episode= 792 ] step = 118\n",
      "[episode= 793 ] step = 29\n",
      "updating target-network parmeters...\n",
      "[episode= 794 ] step = 47\n",
      "[episode= 795 ] step = 107\n",
      "updating target-network parmeters...\n",
      "[episode= 796 ] step = 195\n",
      "updating target-network parmeters...\n",
      "[episode= 797 ] step = 110\n",
      "updating target-network parmeters...\n",
      "[episode= 798 ] step = 163\n",
      "[episode= 799 ] step = 132\n",
      "[episode= 800 ] step = 15\n",
      "updating target-network parmeters...\n",
      "[episode= 801 ] step = 170\n",
      "updating target-network parmeters...\n",
      "[episode= 802 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 803 ] step = 138\n",
      "updating target-network parmeters...\n",
      "[episode= 804 ] step = 160\n",
      "[episode= 805 ] step = 151\n",
      "updating target-network parmeters...\n",
      "[episode= 806 ] step = 107\n",
      "updating target-network parmeters...\n",
      "[episode= 807 ] step = 128\n",
      "[episode= 808 ] step = 129\n",
      "updating target-network parmeters...\n",
      "[episode= 809 ] step = 182\n",
      "updating target-network parmeters...\n",
      "[episode= 810 ] step = 137\n",
      "[episode= 811 ] step = 109\n",
      "updating target-network parmeters...\n",
      "[episode= 812 ] step = 130\n",
      "updating target-network parmeters...\n",
      "[episode= 813 ] step = 174\n",
      "[episode= 814 ] step = 106\n",
      "updating target-network parmeters...\n",
      "[episode= 815 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 816 ] step = 197\n",
      "updating target-network parmeters...\n",
      "[episode= 817 ] step = 149\n",
      "updating target-network parmeters...\n",
      "[episode= 818 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 819 ] step = 153\n",
      "[episode= 820 ] step = 104\n",
      "updating target-network parmeters...\n",
      "[episode= 821 ] step = 88\n",
      "updating target-network parmeters...\n",
      "[episode= 822 ] step = 122\n",
      "updating target-network parmeters...\n",
      "[episode= 823 ] step = 199\n",
      "[episode= 824 ] step = 118\n",
      "updating target-network parmeters...\n",
      "[episode= 825 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 826 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 827 ] step = 130\n",
      "[episode= 828 ] step = 127\n",
      "updating target-network parmeters...\n",
      "[episode= 829 ] step = 129\n",
      "updating target-network parmeters...\n",
      "[episode= 830 ] step = 90\n",
      "[episode= 831 ] step = 113\n",
      "updating target-network parmeters...\n",
      "[episode= 832 ] step = 114\n",
      "[episode= 833 ] step = 111\n",
      "updating target-network parmeters...\n",
      "[episode= 834 ] step = 80\n",
      "[episode= 835 ] step = 109\n",
      "updating target-network parmeters...\n",
      "[episode= 836 ] step = 118\n",
      "[episode= 837 ] step = 110\n",
      "updating target-network parmeters...\n",
      "[episode= 838 ] step = 131\n",
      "updating target-network parmeters...\n",
      "[episode= 839 ] step = 141\n",
      "[episode= 840 ] step = 142\n",
      "updating target-network parmeters...\n",
      "[episode= 841 ] step = 177\n",
      "updating target-network parmeters...\n",
      "[episode= 842 ] step = 48\n",
      "[episode= 843 ] step = 158\n",
      "updating target-network parmeters...\n",
      "[episode= 844 ] step = 146\n",
      "[episode= 845 ] step = 28\n",
      "updating target-network parmeters...\n",
      "[episode= 846 ] step = 75\n",
      "[episode= 847 ] step = 51\n",
      "updating target-network parmeters...\n",
      "[episode= 848 ] step = 142\n",
      "[episode= 849 ] step = 118\n",
      "updating target-network parmeters...\n",
      "[episode= 850 ] step = 101\n",
      "[episode= 851 ] step = 56\n",
      "updating target-network parmeters...\n",
      "[episode= 852 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 853 ] step = 121\n",
      "[episode= 854 ] step = 178\n",
      "updating target-network parmeters...\n",
      "[episode= 855 ] step = 123\n",
      "updating target-network parmeters...\n",
      "[episode= 856 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 857 ] step = 169\n",
      "updating target-network parmeters...\n",
      "[episode= 858 ] step = 132\n",
      "[episode= 859 ] step = 140\n",
      "updating target-network parmeters...\n",
      "[episode= 860 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 861 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 862 ] step = 151\n",
      "updating target-network parmeters...\n",
      "[episode= 863 ] step = 147\n",
      "updating target-network parmeters...\n",
      "[episode= 864 ] step = 193\n",
      "updating target-network parmeters...\n",
      "[episode= 865 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 866 ] step = 149\n",
      "updating target-network parmeters...\n",
      "[episode= 867 ] step = 199\n",
      "[episode= 868 ] step = 162\n",
      "updating target-network parmeters...\n",
      "[episode= 869 ] step = 128\n",
      "updating target-network parmeters...\n",
      "[episode= 870 ] step = 110\n",
      "[episode= 871 ] step = 129\n",
      "updating target-network parmeters...\n",
      "[episode= 872 ] step = 112\n",
      "updating target-network parmeters...\n",
      "[episode= 873 ] step = 148\n",
      "[episode= 874 ] step = 54\n",
      "updating target-network parmeters...\n",
      "[episode= 875 ] step = 185\n",
      "[episode= 876 ] step = 110\n",
      "updating target-network parmeters...\n",
      "[episode= 877 ] step = 88\n",
      "[episode= 878 ] step = 45\n",
      "[episode= 879 ] step = 94\n",
      "updating target-network parmeters...\n",
      "[episode= 880 ] step = 103\n",
      "[episode= 881 ] step = 19\n",
      "updating target-network parmeters...\n",
      "[episode= 882 ] step = 152\n",
      "[episode= 883 ] step = 128\n",
      "updating target-network parmeters...\n",
      "[episode= 884 ] step = 115\n",
      "updating target-network parmeters...\n",
      "[episode= 885 ] step = 141\n",
      "[episode= 886 ] step = 114\n",
      "updating target-network parmeters...\n",
      "[episode= 887 ] step = 139\n",
      "[episode= 888 ] step = 19\n",
      "[episode= 889 ] step = 17\n",
      "updating target-network parmeters...\n",
      "[episode= 890 ] step = 113\n",
      "[episode= 891 ] step = 126\n",
      "updating target-network parmeters...\n",
      "[episode= 892 ] step = 133\n",
      "updating target-network parmeters...\n",
      "[episode= 893 ] step = 123\n",
      "[episode= 894 ] step = 118\n",
      "updating target-network parmeters...\n",
      "[episode= 895 ] step = 164\n",
      "updating target-network parmeters...\n",
      "[episode= 896 ] step = 102\n",
      "updating target-network parmeters...\n",
      "[episode= 897 ] step = 199\n",
      "[episode= 898 ] step = 112\n",
      "updating target-network parmeters...\n",
      "[episode= 899 ] step = 117\n",
      "[episode= 900 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 901 ] step = 117\n",
      "updating target-network parmeters...\n",
      "[episode= 902 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 903 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 904 ] step = 199\n",
      "[episode= 905 ] step = 96\n",
      "updating target-network parmeters...\n",
      "[episode= 906 ] step = 99\n",
      "[episode= 907 ] step = 17\n",
      "[episode= 908 ] step = 62\n",
      "updating target-network parmeters...\n",
      "[episode= 909 ] step = 96\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[episode= 910 ] step = 124\n",
      "updating target-network parmeters...\n",
      "[episode= 911 ] step = 15\n",
      "[episode= 912 ] step = 128\n",
      "updating target-network parmeters...\n",
      "[episode= 913 ] step = 196\n",
      "updating target-network parmeters...\n",
      "[episode= 914 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 915 ] step = 99\n",
      "[episode= 916 ] step = 42\n",
      "[episode= 917 ] step = 82\n",
      "updating target-network parmeters...\n",
      "[episode= 918 ] step = 84\n",
      "updating target-network parmeters...\n",
      "[episode= 919 ] step = 189\n",
      "[episode= 920 ] step = 60\n",
      "[episode= 921 ] step = 29\n",
      "updating target-network parmeters...\n",
      "[episode= 922 ] step = 105\n",
      "[episode= 923 ] step = 45\n",
      "[episode= 924 ] step = 110\n",
      "updating target-network parmeters...\n",
      "[episode= 925 ] step = 112\n",
      "updating target-network parmeters...\n",
      "[episode= 926 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 927 ] step = 126\n",
      "[episode= 928 ] step = 80\n",
      "updating target-network parmeters...\n",
      "[episode= 929 ] step = 176\n",
      "updating target-network parmeters...\n",
      "[episode= 930 ] step = 163\n",
      "updating target-network parmeters...\n",
      "[episode= 931 ] step = 152\n",
      "[episode= 932 ] step = 89\n",
      "updating target-network parmeters...\n",
      "[episode= 933 ] step = 129\n",
      "[episode= 934 ] step = 130\n",
      "updating target-network parmeters...\n",
      "[episode= 935 ] step = 107\n",
      "updating target-network parmeters...\n",
      "[episode= 936 ] step = 136\n",
      "[episode= 937 ] step = 96\n",
      "updating target-network parmeters...\n",
      "[episode= 938 ] step = 186\n",
      "[episode= 939 ] step = 90\n",
      "updating target-network parmeters...\n",
      "[episode= 940 ] step = 192\n",
      "updating target-network parmeters...\n",
      "[episode= 941 ] step = 153\n",
      "updating target-network parmeters...\n",
      "[episode= 942 ] step = 91\n",
      "[episode= 943 ] step = 90\n",
      "updating target-network parmeters...\n",
      "[episode= 944 ] step = 197\n",
      "updating target-network parmeters...\n",
      "[episode= 945 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 946 ] step = 162\n",
      "[episode= 947 ] step = 124\n",
      "updating target-network parmeters...\n",
      "[episode= 948 ] step = 124\n",
      "updating target-network parmeters...\n",
      "[episode= 949 ] step = 133\n",
      "[episode= 950 ] step = 108\n",
      "updating target-network parmeters...\n",
      "[episode= 951 ] step = 146\n",
      "[episode= 952 ] step = 17\n",
      "[episode= 953 ] step = 55\n",
      "updating target-network parmeters...\n",
      "[episode= 954 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 955 ] step = 188\n",
      "updating target-network parmeters...\n",
      "[episode= 956 ] step = 95\n",
      "[episode= 957 ] step = 79\n",
      "updating target-network parmeters...\n",
      "[episode= 958 ] step = 65\n",
      "[episode= 959 ] step = 51\n",
      "[episode= 960 ] step = 128\n",
      "updating target-network parmeters...\n",
      "[episode= 961 ] step = 199\n",
      "updating target-network parmeters...\n",
      "[episode= 962 ] step = 27\n",
      "[episode= 963 ] step = 109\n",
      "updating target-network parmeters...\n",
      "[episode= 964 ] step = 115\n",
      "[episode= 965 ] step = 101\n",
      "[episode= 966 ] step = 14\n",
      "updating target-network parmeters...\n",
      "[episode= 967 ] step = 103\n",
      "updating target-network parmeters...\n",
      "[episode= 968 ] step = 143\n",
      "[episode= 969 ] step = 133\n",
      "[episode= 970 ] step = 30\n",
      "updating target-network parmeters...\n",
      "[episode= 971 ] step = 151\n",
      "updating target-network parmeters...\n",
      "[episode= 972 ] step = 127\n",
      "[episode= 973 ] step = 41\n",
      "updating target-network parmeters...\n",
      "[episode= 974 ] step = 154\n",
      "[episode= 975 ] step = 136\n",
      "updating target-network parmeters...\n",
      "[episode= 976 ] step = 129\n",
      "updating target-network parmeters...\n",
      "[episode= 977 ] step = 109\n",
      "[episode= 978 ] step = 150\n",
      "updating target-network parmeters...\n",
      "[episode= 979 ] step = 35\n",
      "[episode= 980 ] step = 40\n",
      "[episode= 981 ] step = 110\n",
      "updating target-network parmeters...\n",
      "[episode= 982 ] step = 71\n",
      "[episode= 983 ] step = 108\n",
      "updating target-network parmeters...\n",
      "[episode= 984 ] step = 111\n",
      "updating target-network parmeters...\n",
      "[episode= 985 ] step = 177\n",
      "updating target-network parmeters...\n",
      "[episode= 986 ] step = 154\n",
      "[episode= 987 ] step = 95\n",
      "[episode= 988 ] step = 95\n",
      "updating target-network parmeters...\n",
      "[episode= 989 ] step = 111\n",
      "updating target-network parmeters...\n",
      "[episode= 990 ] step = 150\n",
      "[episode= 991 ] step = 116\n",
      "updating target-network parmeters...\n",
      "[episode= 992 ] step = 100\n",
      "[episode= 993 ] step = 103\n",
      "updating target-network parmeters...\n",
      "[episode= 994 ] step = 99\n",
      "updating target-network parmeters...\n",
      "[episode= 995 ] step = 151\n",
      "[episode= 996 ] step = 121\n",
      "updating target-network parmeters...\n",
      "[episode= 997 ] step = 139\n",
      "updating target-network parmeters...\n",
      "[episode= 998 ] step = 111\n",
      "[episode= 999 ] step = 131\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    env = gym.make(ENV)\n",
    "    with tf.Session() as sess:\n",
    "        with tf.variable_scope(\"DQN\"):\n",
    "            DQN = DeepQNetwork( \"DQN\" , env , sess , dueling = False , out_graph = False , out_dqn = True )\n",
    "        with tf.variable_scope(\"Deuling\"):\n",
    "            Dueling = DeepQNetwork(\"Deuling\" , env , sess , dueling = True , out_graph = False , out_dqn = False )\n",
    "        \n",
    "        step_dqn , reward_dqn , loss_dqn = train(DQN , env)\n",
    "        step_dueling , reward_dueling , loss_dueling = train(Dueling , env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
